Make great documentation goal 1.2, and plan the README's reorganisation and the nested-template gaps #33

Open
lilleman wants to merge 966 commits from worktree-todo-relative-dates-exact-counts into main
167 changed files with 645582 additions and 17361 deletions
+162
View File
@@ -0,0 +1,162 @@
name: Packages
on:
workflow_call:
inputs:
tag:
type: string
required: true
secrets:
TAP_GITHUB_TOKEN:
required: true
PACKAGES_GPG_KEY:
required: true
PACKAGES_RSA_KEY:
required: true
DOCKERHUB_TOKEN:
required: true
workflow_dispatch:
inputs:
tag:
description: The released tag to package, such as v0.1.0
type: string
required: true
permissions:
contents: read
concurrency:
group: packages
cancel-in-progress: false
jobs:
packages:
name: Binaries, packages and image for ${{ inputs.tag }}
runs-on: ubuntu-26.04
permissions:
contents: write
packages: write
outputs:
latest: ${{ steps.latest.outputs.latest }}
steps:
- uses: actions/checkout@v7
with:
ref: ${{ inputs.tag }}
path: src
fetch-depth: 0
persist-credentials: false
# Check out the packaging config at the workflow's commit, so a tag cut before the
# config existed can still be packaged.
- uses: actions/checkout@v7
with:
path: tooling
persist-credentials: false
- name: The Go version the gate tests
id: go
run: sed -n 's/^ARG GO_VERSION=/version=/p' tooling/Dockerfile >> "$GITHUB_OUTPUT"
- uses: actions/setup-go@v7
with:
go-version: ${{ steps.go.outputs.version }}
cache: false
- name: Channels holding one version
id: latest
env:
GH_TOKEN: ${{ github.token }}
TAG: ${{ inputs.tag }}
run: |
latest="$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName --jq .tagName)"
if [ "$latest" = "$TAG" ]; then
echo "latest=true" >> "$GITHUB_OUTPUT"
else
echo "skip=--skip=homebrew,scoop" >> "$GITHUB_OUTPUT"
echo "OLDER_TAG=1" >> "$GITHUB_ENV"
fi
- uses: docker/setup-buildx-action@v4
- uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: goreleaser/goreleaser-action@v7
with:
version: v2.18.3
workdir: src
args: release --clean --config ../tooling/release-tooling/goreleaser.yaml ${{ steps.latest.outputs.skip }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }}
- name: The built binary and the published image print the tag
env:
TAG: ${{ inputs.tag }}
run: |
test "$(dist/fejkdata_linux_amd64_v1/fejkdata --version)" = "fejkdata $TAG"
test "$(docker run --rm "ghcr.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG"
dockerhub:
name: Docker Hub image for ${{ inputs.tag }}
needs: packages
runs-on: ubuntu-26.04
permissions:
packages: read
steps:
- uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: docker/login-action@v4
with:
username: ${{ vars.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Copy the GHCR image to Docker Hub, and check it prints the tag
env:
LATEST: ${{ needs.packages.outputs.latest }}
TAG: ${{ inputs.tag }}
run: |
tags=(-t "docker.io/larvit/fejkdata:${TAG#v}")
if [ "$LATEST" = true ]; then
tags+=(-t docker.io/larvit/fejkdata:latest)
fi
docker buildx imagetools create "${tags[@]}" "ghcr.io/larvit/fejkdata:${TAG#v}"
test "$(docker run --rm "docker.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG"
repos:
name: apt, dnf and apk repositories for ${{ inputs.tag }}
needs: packages
if: needs.packages.outputs.latest == 'true'
runs-on: ubuntu-26.04
permissions:
contents: read
pages: write
id-token: write
environment:
name: github-pages
url: ${{ steps.deploy.outputs.page_url }}
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- name: The release's Linux packages
env:
GH_TOKEN: ${{ github.token }}
TAG: ${{ inputs.tag }}
run: gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --dir packages --pattern '*.deb' --pattern '*.rpm' --pattern '*.apk'
- uses: docker/setup-buildx-action@v4
- name: Install fejkdata from each repository, then export them
env:
PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }}
PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }}
TAG: ${{ inputs.tag }}
run: |
build() {
docker buildx build -f release-tooling/package-repos.Dockerfile \
--secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY \
--build-context packages=packages --build-arg "EXPECTED_VERSION=$TAG" "$@" .
}
build --target checks --output type=local,dest=checks
build --target site --output type=local,dest=site
- uses: actions/upload-pages-artifact@v5
with:
path: site
- id: deploy
uses: actions/deploy-pages@v5
+85 -14
View File
@@ -4,6 +4,9 @@ on:
pull_request: pull_request:
push: push:
branches: [main] branches: [main]
workflow_dispatch:
schedule:
- cron: '17 5 * * *'
permissions: permissions:
contents: read contents: read
@@ -11,9 +14,10 @@ permissions:
jobs: jobs:
test: test:
name: vet + fmt + tests name: vet + fmt + tests
runs-on: ubuntu-24.04 if: github.event_name != 'schedule'
runs-on: ubuntu-26.04
steps: steps:
- uses: actions/checkout@v6 - uses: actions/checkout@v7
with: with:
persist-credentials: false persist-credentials: false
fetch-depth: 0 fetch-depth: 0
@@ -26,11 +30,16 @@ jobs:
else else
base='${{ github.event.before }}' base='${{ github.event.before }}'
fi fi
if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum Dockerfile .dockerignore .github data testdata README.md; then if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum Dockerfile .dockerignore .github data release-tooling testdata vulnerability-filing README.md todo.md; then
echo "code=false" >> "$GITHUB_OUTPUT" echo "code=false" >> "$GITHUB_OUTPUT"
else else
echo "code=true" >> "$GITHUB_OUTPUT" echo "code=true" >> "$GITHUB_OUTPUT"
fi fi
if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum release-tooling Dockerfile .dockerignore LICENSE DATA-LICENSES.md .github/workflows/packages.yml .github/workflows/test.yml; then
echo "packaging=false" >> "$GITHUB_OUTPUT"
else
echo "packaging=true" >> "$GITHUB_OUTPUT"
fi
echo "base=$base" >> "$GITHUB_OUTPUT" echo "base=$base" >> "$GITHUB_OUTPUT"
- name: CHANGELOG entry for data changes - name: CHANGELOG entry for data changes
run: | run: |
@@ -48,22 +57,84 @@ jobs:
# failure is version-specific. # failure is version-specific.
- name: Lowest supported Go - name: Lowest supported Go
if: ${{ !cancelled() && steps.changes.outputs.code == 'true' }} if: ${{ !cancelled() && steps.changes.outputs.code == 'true' }}
run: docker build --build-arg GO_VERSION=1.22.12 . run: docker build --build-arg GO_VERSION=1.22.12 --target portable .
- name: Package repositories, signed with throwaway keys
if: ${{ !cancelled() && steps.changes.outputs.packaging == 'true' }}
run: |
docker build -f release-tooling/package-repos.Dockerfile --target throwaway-keys --output "$RUNNER_TEMP/keys" .
PACKAGES_GPG_KEY="$(cat "$RUNNER_TEMP/keys/gpg.asc")" PACKAGES_RSA_KEY="$(cat "$RUNNER_TEMP/keys/rsa.pem")" \
docker build -f release-tooling/package-repos.Dockerfile --target checks \
--secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY .
release: vulnerabilities:
name: Gitea release from CHANGELOG.md name: Known vulnerabilities in this Go version and the required modules
needs: test runs-on: ubuntu-26.04
if: github.event_name == 'push' timeout-minutes: 10
runs-on: ubuntu-24.04
permissions: permissions:
actions: write
contents: write contents: write
pull-requests: write
steps: steps:
- uses: actions/checkout@v6 - uses: actions/checkout@v7
with: with:
persist-credentials: false persist-credentials: false
- run: python3 release-tooling/publish_release.py - name: Fetch govulncheck's report from vuln.go.dev
run: docker build --no-cache-filter vulnerabilities --target vulnerability-report --output "$RUNNER_TEMP/vulnerabilities" .
- name: File each one in todo.md
id: file
run: python3 vulnerability-filing/file_vulnerabilities.py "$RUNNER_TEMP/vulnerabilities/report.json" todo.md
- name: Propose the filed items
if: failure() && steps.file.outcome == 'failure' && github.ref == 'refs/heads/main'
env: env:
GITEA_API_URL: ${{ github.api_url }} GH_TOKEN: ${{ github.token }}
GITEA_REPOSITORY: ${{ github.repository }} run: |
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }} if git diff --quiet todo.md || [ "$(gh pr list --head vulnerabilities --state open --json number --jq length)" != 0 ]; then
exit 0
fi
git config user.name 'github-actions[bot]'
git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
git switch -C vulnerabilities
git commit -qam "File the known vulnerabilities govulncheck reports"
git push -qf "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" vulnerabilities
gh pr create --head vulnerabilities --title "File the known vulnerabilities govulncheck reports" \
--body "govulncheck reports known vulnerabilities in the Go version or a required module, and no release is cut until they are gone. This files them atop the next release in todo.md, scored 0.0 until rank_todo.py scores them; each item names the bump that fixes it."
# A pull request the workflow's token opens starts no workflow.
gh workflow run test.yml --ref vulnerabilities
release:
name: Release from CHANGELOG.md
needs: [test, vulnerabilities]
if: github.event_name == 'push'
runs-on: ubuntu-26.04
permissions:
contents: write
outputs:
tag: ${{ steps.publish.outputs.tag }}
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- id: publish
run: python3 release-tooling/publish_release.py
env:
FORGE_API_URL: ${{ github.api_url }}
FORGE_REPOSITORY: ${{ github.repository }}
FORGE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SHA: ${{ github.sha }} SHA: ${{ github.sha }}
packages:
needs: release
if: needs.release.outputs.tag != ''
permissions:
contents: write
packages: write
pages: write
id-token: write
uses: ./.github/workflows/packages.yml
with:
tag: ${{ needs.release.outputs.tag }}
secrets:
TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }}
PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }}
PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+126 -16
View File
@@ -1,27 +1,137 @@
# Rules # Rules
- Go runs only through `docker compose run --rm <test|vet|fmt|build|cyclo>`; the merge gate is `docker build .` plus CI's changelog check. - Go runs only through `docker compose run --rm <test|vet|fmt|build|cyclo|generate|vulncheck>`; the merge gate is what the README's Development section lists.
- Tests first, in their own commit; the implementation follows in the next. A re-pin of seeded output or of `testdata/shipped_shape.txt` is its own commit. - Tests first, in their own commit; the implementation follows in the next. A re-pin of seeded output or of `testdata/shipped_shape.txt` is its own commit.
- A change under `data/` or to the shape pin adds its `CHANGELOG.md` entry under `Unreleased` in the same PR, and so does a change to a flag, an exit code, an exported name, a fence, a builtin or the lowest Go; what is major is the README's Versioning table. - A change under `data/` or to the shape pin adds its `CHANGELOG.md` entry under `Unreleased` in the same PR, and so does a change to a flag, an exit code, an exported name, a fence, a builtin or the lowest Go; what is major is the README's Versioning table.
- One-line commit messages: no ticket prefix, no repo name, no authorship trailers. - One-line commit messages: no ticket prefix, no repo name, no authorship trailers.
- Merges are fast-forward only, which `tea pr merge` cannot spell — every style it - GitHub is where the project lives, so `gh` is the tool: merges are fast-forward
offers fails with "is it still open?", as does a merge whose required checks are only, `gh pr merge --rebase` where the branch is already on top of main.
still pending, so read the checks before believing the style is the problem. Use - CodeRabbit skips every pull request until the repository has 10 GitHub stars: check
`tea api /repos/{owner}/{repo}/pulls/<n>/merge -f Do=fast-forward-only`. its status once when a PR is marked ready, answer a review if one is there, and never
- Gitea queues Actions runs that `/repos/{owner}/{repo}/actions/tasks` does not wait for one.
list, so an empty task list says nothing about whether CI ran; read - From 0.2.0, README goal 3 is gated at 7.0 on the `comprehension-panel` skill, and below it
`/repos/{owner}/{repo}/commits/<sha>/status`. nothing else merges — no feature, no category, no data — bar a security fix, a
- README goal 2 is gated at 7.0 on the `comprehension-panel` skill, and below it dependency bump, a repair of behaviour the README documents, the infrastructure
nothing else merges — no feature, no category, no data — bar a security fix and a the round itself runs on, and a chunk the comprehension-floor decision lists as
dependency bump. `todo.md` carries the round: the nine-seat findings run at depth 1, shipped under the floor. `todo.md` carries the round: a PR per item it plans, the nine-seat
a PR per item it names, then the run again, until the score passes. At or above 7.0 findings at depth 1 or a restructure the maintainer approved, then the nine-seat run
again, until the score passes. At or above 7.0
every pull request scores with the four-seat run and answers it in one run. No merge every pull request scores with the four-seat run and answers it in one run. No merge
lowers the last score. lowers the last score its own panel recorded: the nine-seat round and the four-seat
run are two ratchets, never compared with each other. A lower score blocks the merge
even where it may be panel noise: fix what the seats name until the score rises. A PR
that only records a run's scores and files what its seats name merges even when a
score fell. Through 0.1.0, a merge is blocked only by a paired ruling that rules worse.
- Hard tabs. No comment by default; delete a restatement, a rationale, history, or a file preamble. - Hard tabs. No comment by default; delete a restatement, a rationale, history, or a file preamble.
- One spelling per result: reject the other at `New`, and let the error name the spelling to use.
- A standing choice a reader would relitigate goes under Decisions in the README, not in a comment.
- Prose naming a source states what that source states: read the register's own field - Prose naming a source states what that source states: read the register's own field
before paraphrasing it, and name the register you read before claiming none before paraphrasing it, and name the register you read before claiming none
publishes a spelling. publishes a spelling.
- A README example is a `json` block that loads and renders as a category; `readme_test.go` runs every one. - A README example is a `json` block that loads and renders as a category; `readme_test.go` runs every one.
- Cyclomatic complexity is gated at 14: the table-shaped dispatches (`linkParent`, `compileTemplate`, `renderEdges`) sit at it and stay whole; a function that would pass it is decomposed. - Cyclomatic complexity is gated at 14: split a function that would go over it. A table-shaped dispatch, a switch with one case per kind of its input, may sit at 14 and stays whole.
# Decisions
In [docs/decisions.md](docs/decisions.md):
- GitHub is canonical, and the module path names it
- The Gitea copy stays, as a pull mirror of GitHub
- The vocabulary sits below `doc.go`'s package clause, not in the package doc
- Options and fields share one namespace
- `{a|b}` stays beside nested choices
- Flags follow getopt_long
- A struct tag is a template by its shape
- The CLI renders exactly the template on its stdin, and adds nothing
- The CLI reads only the library's public API
- A template that does not compile is misuse (exit 2), including a reference that resolves to nothing, save a lone reference under the CLI's `--format` until item 129
- Whitespace around a JSON object, array, string or null is dropped, and a bare number or boolean is a format string
- `FakeTemplate` and `NewTemplate` both stay
- The shipped data is embedded, not discovered
- The shipped data is Go packages a library imports by choice, and the CLI carries every one, from 0.1.0
- A shipped package's folder spells its namespace, its Go name is that namespace in lower case without separators, and its manifest mounts it with `namespace`
- A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads
- Every `{…}` draws afresh, and only a name keeps a pick
- Reference sigils follow the filesystem
- A change to what exists is a major; a minor only adds
- An error is a contract by what it names, not its bytes
- A load reports every mistake at once
- Formats are checked after all data is loaded, so their errors name the category
- Raising the lowest supported Go is a major
- The format check runs on the latest Go only
- The changelog heading is the one spelling of a release; CI cuts the tag
- A `--data-path` override rebinds every reference to the category it replaces
- The repeat cap bounds renders, not bytes
- 64-bit targets only
- A constant zero divisor is a load error; in a string column a divisor that is not constant prints `Inf`
- Samples say what they emit, transforms what they do
- A record is a template seen as columns; a Go struct is the one second schema
- A struct's records follow Go's field access, and compile on first use
- A category may reference itself, and a record's fences run at load
- A record's column set is fixed before the first draw
- Null is a `null` item, not a rate
- A typed column holds one value, not composed text
- A column that only reads one reference or name is the column it reads
- A typed column's calc is refused unless proven
- `Column` carries text, not a Go value
- A unit takes the stem of what it is, and a file the stem of the units it holds
- One name, one meaning
- The performance gate asserts allocations, not wall-clock time
- Rows live in a TSV, the shape in JSON
- A selector is bracketed, and a dot inside it is literal
- `parent` names the link column and the table alike
- After a row, a path names a column or a linked table
- A selected row is fixed, not drawn
- `..` steps up to the parent row, and a step down after it draws afresh, or once per name's pick
- A table selects by one key column, a code, never a free-form name
- A parent row with no child row is a load error
- A table is a record of string columns
- The key map is built at load, the rest on first draw
- Two categories may name one TSV
- A cell may read a reference, and each row its own
- A path is walked once without drawing before it is walked for real
- A country's postal codes and streets are siblings under its locality
- A locale package imports its country's geo package, and its `Modules()` returns both
- The default embed holds every Swedish postort the import can place, with all its streets, and, of the US places that hold most of a postal code's land, every state capital, every place of 100,000 or more and a population-weighted tenth, each with a quarter of its streets, at least ten
- A locale's `address` binds its country's record once and prints it whole
- A postort's kommun comes from its name, its tätort or its codes, never from distance
- A highway designation is not a street, and a US postal code belongs to the place holding most of its land inside places
- `--list` stays a plain list of paths
- A layout is always quoted
- A title is a table under `sex`
- A table owns the spelling of a selector on it
- No builtin reads the clock, so a date is bounded by days, never by an age
- `misc` is what every locale shares
- A register's canonical spelling loses to the one its domain writes
- `misc.territory` is the spine, and a `misc` table naming a territory links to it
- A table whose register publishes no frequency draws evenly
- `misc.timezone` weighs a zone by the people living in it
- `misc.car` is one flat table, not a make linked to its models
- `misc.territory` names its sovereign in a column, and there is no `misc.country` table
- `misc.loglevel` is a table keyed by POSIX's keyword, carrying the code
- `misc.tld` keys carry the leading dot, where other tables key on a bare code
- `misc.tld` is a table of its own, and `misc.territory.tld` stays a column
- `misc.territory` carries a currency code, it does not link to `misc.currency`
- An extension may name two media types
- The Swedish ids draw Skatteverket's test series
- The US given names come from a mirror of the SSA file
- `List` advertises direct descents only
- A path draws through its compiled steps
- A name lives in the category binding it, or in the repeat or choice item binding it, and is drawn on its first read
- comprehension floor: every dimension and the overall at 7.0 or above, from 0.2.0
- The template engine stays the root package until item 107; what reads no engine type, a table's rows included, sits in `internal/`
- Goal 5.1's `{…}` covers `{{`, `}}` and a lone `}`, so the goal names no escape
- Goal 7.2 holds while a first template can be written with no escape; a literal brace still takes `{{` or `}}`
- The locales are Ethnologue 2026's ten most-spoken languages, each in the country with the most speakers of it, the Nordic countries and Germany
- A country with no open address register covering it whole ships its places and postal codes without streets
- The apt, dnf and apk repositories are static files on GitHub Pages, holding the latest release only
- The image is pushed to GHCR and copied to Docker Hub, whose name the README shows
- A known vulnerability in the Go version or a required module blocks a release, even in code fejkdata never calls, and blocks no merge
# Scoring run
The project values for `~/.claude/skills/comprehension-panel/scoring-run.md`:
- `{language}`: Go
- `{kind}`: a Go library and CLI that renders fake data from JSON templates
- `{domain}`: fake test-data generation and template languages
- `{domain docs}`: the documentation of any fake-data library, such as Faker
- `{3am question}`: a seeded render of `sv_SE.address` prints the postal code of one postort beside a street of another
+96 -82
View File
@@ -8,85 +8,99 @@ replacement, and each removed path, column or flag.
### Added ### Added
- First release: the CLI, the library and the shipped data set. - Prebuilt binaries for Linux, macOS and Windows on amd64 and arm64, packages for apt,
- Table categories: a category JSON naming a `rows` TSV beside it, with the options dnf, apk, Homebrew and Scoop, and the image `larvit/fejkdata` on Docker Hub, also at `ghcr.io/larvit/fejkdata`, so
`key`, `name`, `weight` and `parent`; a path selects a row by key or name, the CLI installs without Go. 0.1.0 ships them too; the README's
`misc.territory[SE]`, and descends to a linked table by name; linked tables draw [Install](https://github.com/larvit/fejkdata#install) lists each command.
consistently within one render and draw group. `rows` is an option, so no
template may carry a field of that name. A `name` without a `key` resolves ## [0.1.0] - 2026-10-13
inside the table's `parent`. Refused at `New`: a `name` without a `key` or a
`parent`, a name repeating inside one parent row, a name spelling another row's key, a table named like a column of any The first release: a Go library and a CLI that render locale-aware fake data from JSON
table above it, a table whose format or cell references its own family, and, templates, and the data they ship with. The
within one render and draw group, a path drawing a table another path selects a [README](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md) documents each part.
row of, or two paths pinning different rows of one table. Install the CLI with `go install github.com/larvit/fejkdata/cmd/fejkdata@v0.1.0`, or the
- `New` refuses a root choice of templates sharing one format and one set of string library with `go get github.com/larvit/fejkdata@v0.1.0`. Before 1.0.0 a minor may break,
fields, naming the rows TSV to write instead. and any release may change what a seed draws, so pin one exact version
- `misc.territory`, `misc.currency`, `misc.language`, `misc.httpstatus` and ([Versioning](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#versioning)).
`misc.mimetype` are tables. `misc.territory` is every ISO 3166-1 territory that has
a capital, a currency and a TLD, with the columns `calling-code`, `capital`, ### Added
`country`, `currency`, `flag`, `languages`, `numeric` and `tld` added, `country`
holding the alpha-2 code of the sovereign state it belongs to and its own where it - The CLI, `fejkdata`, renders exactly the template on its stdin and adds nothing:
is one; `misc.currency` the `echo '{/sv_SE.person} hihi' | fejkdata` prints e.g. `Sara Eriksson hihi`. `--seed`
current ISO 4217 currencies with a minor unit, with `decimals` and `numeric` repeats a run's output while the version and data stay the same, and without it each
added, and its symbols from CLDR; `misc.language` every ISO 639-2 entry carrying a run takes a new seed; `-n` renders the template that many times, and `--list`
639-1 code, with a `code3` column holding its 639-2/T code; `misc.httpstatus` each prints the paths the data offers.
code the IANA registry lists with a plain reason phrase; and `misc.mimetype` each `--data-path` layers a directory of your own data over the shipped set, and
type mime-db records as IANA-registered and gives a filename extension. `--no-shipped-data` loads that directory alone. `--format json|ndjson|csv|sql` writes
`DATA-LICENSES.md` lists each source. a record's columns, one record per row. The CLI exits `0` on success, `1` on a runtime
- `misc.timezone`, `misc.car` and `misc.useragent` are tables too. `misc.timezone` error such as data that fails to load, and `2` on misuse.
is every zone tzdb `zone.tab` gives a shipped territory, with `offset` holding the - The library, `github.com/larvit/fejkdata`, for Go 1.22 or later on a 64-bit target.
zone's standard UTC offset and `territory` linking to `misc.territory`, so `New` loads the modules `WithDataFS` names and the folders `WithDataPath` names, drawing
`misc.territory[SE].timezone` draws `Europe/Stockholm`; zones of a territory from the seed `WithSeed` gives. Without a seed, `New` fails and names `WithSeed`;
`misc.territory` does not ship, Antarctica's among them, and the constant `UTC` without data, it fails and names `WithDataFS`. The same seed repeats the draws
are gone; spell that one as the text `"UTC"`. A `weight` column carries the while the version and data stay the same. `Fake` renders a path, and `FakeTemplate`
population GeoNames records in each zone, so a draw favours the zones people live and `NewTemplate` render a template. `FakeRecord`, `FakeRecordTemplate` and
in. `misc.useragent` is the top-user-agents desktop and mobile lists with `browser`, `NewRecordTemplate` render a record, whose `JSON`, `CSVHeader`, `CSVLine` and
`device` and `os` columns. `misc.car` keeps its makes and models in a `make` and a `SQLInsert` methods write its columns as the CLI does. `FakeStruct` fills a struct's
`model` column. `fake:"…"` tagged fields. `List` gives the paths `Fake` renders, direct descents only,
- `misc.httpmethod`, `misc.protocol` and `misc.port` are IANA's registries as tables. and `IsTemplate` tells an inline template from a path. Data that fails to load gives
`misc.httpmethod` is the eight methods RFC 9110 defines and PATCH, carrying an error matching `ErrLoad`, and a record with no fields gives one matching
`safe` and `idempotent` as `true` or `false`; `misc.protocol` the keyword, its `ErrNoColumns`.
`number` and its `name`, selectable by either; `misc.port` renders a TCP port number, - Package `github.com/larvit/fejkdata/cli` is the `fejkdata` command:
with the IANA `service` name beside it, keeping the first service the registry `cli.Main(modules...)` runs it over the modules given, so
describes for a port so a number selects one row. `cli.Main(append(data.Modules(), mydata)...)` builds a CLI carrying your own data.
- `misc.tld` is the IANA root zone as a table: every delegated TLD keyed with its That CLI names itself by the file it runs from, and its `--version` prints its own
leading dot, `.se` and `.xn--p1ai`, the register's `type` beside it, and a `unicode` module's version before fejkdata's.
column holding the form the register displays, so `misc.tld[.рф]` selects the row - The template language
`.xn--p1ai` renders. A delegation the register marks "Not assigned" does not ship. ([Data format](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data-format)).
- `misc.loglevel` is the eight syslog severities: the `keyword` a configuration writes, Every value in a data file is a string, a choice, a template, a table, a number or a
`err` and `info`, keyed by the numerical `code` a PRI encodes and selectable by boolean. A `{…}` token reads a sibling field, a name, or a reference from the data
either, with the `severity` RFC 5424 spells beside it. root, the file's folder or the folder above (`{/sv_SE.person.last}`, `{.x}`, `{..x}`),
- `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`, or it calls a builtin. Every `{…}` draws afresh, and a name is the one way to keep a
`locality`, `postal-code` and `street`, weighted by population and address counts pick: `{/sv_SE.person as p}{p.first} {p.last}` is one person. The builtins draw check
and built from SCB, GeoNames, Trafikverket NVDB and the US Census Bureau, and an digits (`luhn`, `mod11`, `ean`), samples (`digits`, `int`, `float`, `uuid`, `ulid`,
`address` record over one consistent draw of them. `sv_SE.address` and `iban`, `date`, `time` and more), a counter (`seq`), arithmetic (`calc`) and
`en_US.address` read those records, so `en_US.address.street` no longer carries transforms (`lowercase`, `uppercase`, `ascii`). `New` refuses data that breaks the
`name` and `suffix`, and a locale folder loads only beside `geo`. grammar, could mean two things or cannot render a valid value.
- `{date(from,to,'layout')}` and `{time('layout')}`: a second between two days, or - Records ([Records](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#records)):
within one, in a single-quoted Go layout, drawn in UTC; `from` may equal `to`. a template seen as columns. A column is a string unless it declares a `datatype`,
`sv_SE.date`, `en_US.date`, `sv_SE.time` and `en_US.time` render through them, so `integer`, `number` or `boolean`, and a `null` item draws it as null.
`date.year`, `date.month`, `date.day`, `time.hour`, `time.minute`, `time.minute.t`, - Tables ([Table](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#table)): a
`time.sec` and `time.ampm` are no longer paths — a part of a date is now its own category naming a `rows` TSV beside it, with the options `key`, `name`, `weight` and
`{date(…,'2006')}`. `misc.datetime` is an RFC 3339 instant. `parent`. A path selects a row by key or name, `misc.territory[SE]`, descends to a
- `sex`, `first-name` and `last-name` tables in `sv_SE` and `en_US`, weighted by linked table, `geo.SE.locality[Lund].street`, and steps up to the parent row,
bearers from SCB, the SSA and the Census Bureau; `first-name` links to `sex`, and a `{/misc.timezone[Europe/Oslo]..territory.name}`.
name both sexes carry is a row under each. `person` reads them, so its columns are - Data sources ([Data](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data)):
`first`, `last`, `prefix` and `sex`; `person.femalefirst` and `person.malefirst` are a directory is a namespace, and each JSON file in it a category. A `.fejkdata.json`
no longer paths — draw `sex[f].first-name` and `sex[m].first-name` instead. manifest at a source's root may carry a `namespace`, the dot path the source mounts
`en_US.title` links to `sex` as well, so `en_US.person.prefix` draws `Mr` or `Ms` under, and an `index`, with which each category loads on the first call reaching it.
without contradicting the record's `sex`; `sv_SE.title` is an unsexed table of the - The shipped data, in Go packages that each export `FS` and `Modules()`:
same shape. `github.com/larvit/fejkdata/data/en_US` (package `enus`), `…/sv_SE` (`svse`),
- `sv_SE.personnummer` and `sv_SE.samordningsnummer`, Skatteverket's test series `…/misc`, `…/geo/SE` (`geose`) and `…/geo/US` (`geous`). `data.Modules()`, from
from a `sv_SE.birth-number` table under `sex`, in place of `sv_SE.ssn`, whose `github.com/larvit/fejkdata/data`, lists every one, and the CLI carries them all.
`ssn.mmdd`, `ssn.mmdd.m` and `ssn.mmdd.d` go with it. `en_US.ssn` now draws the [`DATA-LICENSES.md`](https://github.com/larvit/fejkdata/blob/v0.1.0/DATA-LICENSES.md)
ranges the SSA assigns and carries the columns `area`, `group` and `serial`, so names each source.
`--format csv en_US.ssn` writes a header where it used to fail; `en_US.itin` is new - `sv_SE` and `en_US`: an `address`, a `person` and more per locale. `sex`,
and carries the same three. `person.prefix` is null where a person has no title, `first-name` and `last-name` are tables weighted by bearers, from SCB, the SSA and
where it used to be an empty string, so `--format sql` writes `NULL` and a `string` the Census Bureau. A person's first name and sex agree, and so does an `en_US`
struct field reading it becomes `*string`. person's title; `person.prefix` is null where a person has no title.
- `ErrNoColumns` is exported, so a caller can tell the one record fence a path can `sv_SE.personnummer` and `sv_SE.samordningsnummer` draw Skatteverket's test series.
answer from the rest. `en_US.ssn` and `en_US.itin` draw the ranges the SSA and the IRS assign, with the
- An error names a spelling that runs: a layout is named single-quoted and free of columns `area`, `group` and `serial`, and `en_US.phone` draws an exchange NANPA may
its own quotes, a row of a table with no key is named as the path that selects it, assign.
`sv_SE.sex[f].first-name[Kim]`, and a category with no columns names the record - `misc`: IANA's registries of HTTP methods, protocols, ports and root-zone TLDs, the
that gives it one. HTTP status codes, ISO 3166-1 territories, ISO 4217 currencies with their CLDR
symbols, ISO 639 languages, the syslog severities, mime-db's media types, tzdb's
zones weighted by the people living in each, user agents and cars, all as tables;
and coordinates, credit card numbers, RFC 3339 instants, emoji, MAC addresses,
ObjectIds and UUIDs.
- `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`,
`locality`, `postal-code` and `street`, weighted by population and address counts
and built from SCB, GeoNames, Trafikverket NVDB, the US Census Bureau and Wikidata,
and an `address` record that draws a street and a postal code inside one
locality; `sv_SE.address` and `en_US.address` print it. Every Swedish postort
(postal town) the import can place ships with all its streets. A US place ships
only if it is incorporated and holds most of some postal code's land. Of those
places, every state capital but Honolulu ships, as does every place of 100,000
people or more and a tenth of all of them drawn by population. Each US place
carries a quarter of its streets, at least ten.
+2 -2
View File
@@ -9,12 +9,12 @@ Every shipped dataset, its source, its licence and the attribution it asks for.
| `geo/SE/locality.tsv`, `postal-code.tsv` | [GeoNames](https://www.geonames.org/) postal codes for SE; populations from SCB tätorter 2023 | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/); CC0 1.0 | "Postal codes from GeoNames, www.geonames.org" | `data-import/geo-se.py` | | `geo/SE/locality.tsv`, `postal-code.tsv` | [GeoNames](https://www.geonames.org/) postal codes for SE; populations from SCB tätorter 2023 | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/); CC0 1.0 | "Postal codes from GeoNames, www.geonames.org" | `data-import/geo-se.py` |
| `geo/SE/region.tsv`, `municipality.tsv` | [SCB](https://www.scb.se/) län and kommun codes 2026 and population 2024 | [CC0 1.0](https://creativecommons.org/publicdomain/zero/1.0/) | none required | `data-import/geo-se.py` | | `geo/SE/region.tsv`, `municipality.tsv` | [SCB](https://www.scb.se/) län and kommun codes 2026 and population 2024 | [CC0 1.0](https://creativecommons.org/publicdomain/zero/1.0/) | none required | `data-import/geo-se.py` |
| `geo/SE/street.tsv` | [Trafikverket NVDB](https://www.trafikverket.se/) Gatunamn, through the open API | CC0 1.0 | none required | `data-import/geo-se.py` | | `geo/SE/street.tsv` | [Trafikverket NVDB](https://www.trafikverket.se/) Gatunamn, through the open API | CC0 1.0 | none required | `data-import/geo-se.py` |
| `geo/US/region.tsv`, `municipality.tsv`, `locality.tsv` | [Census Bureau](https://www.census.gov/) Gazetteer 2026 and population estimates 2025 | [public domain](https://www.usa.gov/government-works) | none required | `data-import/geo-us.py` | | `geo/US/region.tsv`, `municipality.tsv`, `locality.tsv` | [Census Bureau](https://www.census.gov/) Gazetteer 2026 and population estimates 2025; which places are state capitals from [Wikidata](https://www.wikidata.org/) | [public domain](https://www.usa.gov/government-works); CC0 1.0 | none required | `data-import/geo-us.py` |
| `geo/US/postal-code.tsv`, `street.tsv` | Census Bureau ZCTA to place relationships 2020 and TIGER/Line 2025 address ranges and feature names | public domain | none required | `data-import/geo-us.py` | | `geo/US/postal-code.tsv`, `street.tsv` | Census Bureau ZCTA to place relationships 2020 and TIGER/Line 2025 address ranges and feature names | public domain | none required | `data-import/geo-us.py` |
| `sv_SE/first-name.tsv`, `last-name.tsv` | [SCB](https://www.scb.se/) names with at least two bearers, 31 December 2022 | CC0 1.0 | "Källa: SCB" | `data-import/names-se.py` | | `sv_SE/first-name.tsv`, `last-name.tsv` | [SCB](https://www.scb.se/) names with at least two bearers, 31 December 2022 | CC0 1.0 | "Källa: SCB" | `data-import/names-se.py` |
| `en_US/first-name.tsv` | [SSA](https://www.ssa.gov/oact/babynames/) baby names, births 1930 to 2020, through [hackerb9/ssa-baby-names](https://github.com/hackerb9/ssa-baby-names) | public domain | none required | `data-import/names-us.py` | | `en_US/first-name.tsv` | [SSA](https://www.ssa.gov/oact/babynames/) baby names, births 1930 to 2020, through [hackerb9/ssa-baby-names](https://github.com/hackerb9/ssa-baby-names) | public domain | none required | `data-import/names-us.py` |
| `en_US/last-name.tsv` | Census Bureau surnames occurring 100 or more times, 2010 | public domain | none required | `data-import/names-us.py` | | `en_US/last-name.tsv` | Census Bureau surnames occurring 100 or more times, 2010 | public domain | none required | `data-import/names-us.py` |
| `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv`, `misc/car.tsv` | curated; Skatteverket's test birth numbers are facts, and `car.tsv` waits for an international source (goal 12) | — | — | — | | `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv`, `misc/car.tsv` | curated; Skatteverket's test birth numbers are facts, and `car.tsv` waits for an international source (goal 13) | — | — | — |
| `misc/currency.tsv` | [datasets/currency-codes](https://github.com/datasets/currency-codes); symbols from [Unicode CLDR](https://github.com/unicode-org/cldr) `en.xml` and `root.xml` | PDDL 1.0; [Unicode License v3](https://www.unicode.org/license.txt) | CLDR: "Copyright © 1991-2025 Unicode, Inc. Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries." | `data-import/currency.py` | | `misc/currency.tsv` | [datasets/currency-codes](https://github.com/datasets/currency-codes); symbols from [Unicode CLDR](https://github.com/unicode-org/cldr) `en.xml` and `root.xml` | PDDL 1.0; [Unicode License v3](https://www.unicode.org/license.txt) | CLDR: "Copyright © 1991-2025 Unicode, Inc. Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries." | `data-import/currency.py` |
| `misc/httpmethod.tsv` | [IANA HTTP Method Registry](https://www.iana.org/assignments/http-methods/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpmethod.py` | | `misc/httpmethod.tsv` | [IANA HTTP Method Registry](https://www.iana.org/assignments/http-methods/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpmethod.py` |
| `misc/httpstatus.tsv` | [IANA HTTP Status Code Registry](https://www.iana.org/assignments/http-status-codes/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpstatus.py` | | `misc/httpstatus.tsv` | [IANA HTTP Status Code Registry](https://www.iana.org/assignments/http-status-codes/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpstatus.py` |
+33 -9
View File
@@ -1,19 +1,43 @@
# The merge gate: `docker build .` fails if vet, formatting or tests fail, and CI ARG GO_VERSION=1.27.2
# runs exactly this. Day-to-day, prefer `docker compose run` (bind-mounts source, FROM golang:${GO_VERSION} AS portable
# no rebuilds). GO_VERSION defaults to the latest stable Go; override it to test
# the lowest supported version: docker build --build-arg GO_VERSION=1.22.12 .
ARG GO_VERSION=1.26.4
FROM golang:${GO_VERSION}
WORKDIR /app WORKDIR /app
# Module layer cached separately from source.
COPY go.mod ./ COPY go.mod ./
RUN go mod download RUN go mod download
COPY . . COPY . .
RUN go vet ./... && \ RUN go vet ./... && \
go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test . && \ go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test . && \
{ unformatted="$(gofmt -l .)"; test -z "$unformatted" || \
{ echo "unformatted files:"; echo "$unformatted"; exit 1; }; } && \
go test -race ./... go test -race ./...
FROM golang:${GO_VERSION} AS vulnerabilities
WORKDIR /app
COPY . .
RUN go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan module -format json > /report.json
FROM scratch AS vulnerability-report
COPY --from=vulnerabilities /report.json /
FROM python:3.14.8-slim AS release-tooling
COPY release-tooling/ /release-tooling/
RUN python3 -m unittest discover -s /release-tooling -p '*_test.py' && touch /tested
FROM python:3.14.8-slim AS vulnerability-filing
COPY vulnerability-filing/ /vulnerability-filing/
# The filer's test files an item into the plan itself.
COPY todo.md /todo.md
RUN python3 -m unittest discover -s /vulnerability-filing -p '*_test.py' && touch /tested
FROM goreleaser/goreleaser:v2.18.3 AS packaging
WORKDIR /check
COPY release-tooling/goreleaser.yaml ./
RUN git init -q && git remote add origin https://github.com/larvit/fejkdata.git && goreleaser check --config goreleaser.yaml && touch /tested
FROM portable
# BuildKit skips a stage nothing copies from.
COPY --from=release-tooling /tested /tmp/release-tooling-tested
COPY --from=vulnerability-filing /tested /tmp/vulnerability-filing-tested
COPY --from=packaging /tested /tmp/packaging-tested
RUN unformatted="$(gofmt -l .)"; test -z "$unformatted" || \
{ echo "unformatted files:"; echo "$unformatted"; exit 1; }
+1
View File
@@ -1,6 +1,7 @@
MIT License MIT License
Copyright (c) 2026 Timewave Copyright (c) 2026 Timewave
Copyright (c) 2026 Larvit
Permission is hereby granted, free of charge, to any person obtaining a copy Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal of this software and associated documentation files (the "Software"), to deal
+775 -861
View File
File diff suppressed because it is too large Load Diff
+117
View File
@@ -0,0 +1,117 @@
package fejkdata
import (
"fmt"
"github.com/larvit/fejkdata/internal/builtinfunc"
"github.com/larvit/fejkdata/internal/grammar"
"github.com/larvit/fejkdata/internal/invariant"
)
// arm is one alternative of a {a|b} token or one operand, split into the head,
// whose node `template.startOf` finds, and the tail of a dotted path into it.
type arm struct {
spelling string // as written, for messages
head string
tail []string
levels []pickKey // the key of each level the steps pass, from the head they start at to the leaf; a step's at indexes it
steps []pathStep
leaves []node // every node the path may land on, one per variant it passes
kind armKind
named *nameBinding // namedRead: the binding of the name it reads through
}
// armKind is how expand reads an arm, fixed at compile.
type armKind uint8
const (
freshRead armKind = iota // drawn afresh at every read
namedRead // read through a name, kept in its pick
)
// splitArm splits one name into head and tail. refs maps a reference to what
// resolveRefs resolved it to; before that, a reference is whole.
func splitArm(name string, refs map[string]resolvedRef) arm {
if grammar.IsRef(name) {
if r, resolved := refs[name]; resolved {
return pathArm(name, r.head, r.tail)
}
return pathArm(name, name, nil)
}
segs, err := grammar.SplitPath(name)
if err != nil || len(segs) == 1 {
return pathArm(name, name, nil)
}
return pathArm(name, segs[0], segs[1:])
}
func pathArm(name, head string, segs []string) arm {
return arm{spelling: name, head: head, tail: segs, levels: levelKeys(append([]string{head}, segs...), 1)}
}
// key is the key of the leaf a lands on, the one key every way of writing this read shares.
func (a arm) key() pickKey { return a.levels[len(a.levels)-1] }
func checkSegments(a arm) error {
if len(a.tail) == 0 {
return nil
}
return grammar.CheckSegments(append([]string{a.head}, a.tail...))
}
// op is one compiled unit of a format string: a literal run, a name read,
// or a builtin already prepared with its args.
type op struct {
grammar.Token
arms []arm // grammar.PathRead: the '|' alternatives, split into head and tail once
call builtinfunc.Call
// operands are the fields the builtin reads, in the order its operands func
// fixed; expand reads them before the call. nil for a builtin that reads none.
operands []arm
}
func (t *template) compileArms(names []string, targets map[*nameBinding]nameTarget) ([]arm, error) {
if len(names) == 0 {
return nil, nil
}
arms := make([]arm, len(names))
for i, name := range names {
a, err := t.compileArm(name, targets)
if err != nil {
return nil, err
}
arms[i] = a
}
return arms, nil
}
// compileArm compiles one read into a path: from the head it names, or, for a read through a
// name, from what that name binds.
func (t *template) compileArm(name string, targets map[*nameBinding]nameTarget) (arm, error) {
a := splitArm(name, t.refs.byName)
if !t.isName(a.head) {
start := t.startOf(a.head)
if start == nil {
panic(invariant.Broken("{%s} reads a head nothing resolved", a.spelling))
}
w := compilePath(start, a.tail)
a.steps, a.leaves = w.steps, w.leaves
return a, nil
}
b := t.nameScope.lookup(a.head)
if grammar.HasSelector(a.tail) {
if b.bindsField() {
return a, fmt.Errorf("a path through name %q may not select a row; bind the row to a name of its own", a.head)
}
return a, fmt.Errorf("a path through name %q may not select a row; read the row without the name, {%s}, or bind it to a name of its own", a.head, grammar.JoinSegments(append([]string{b.ref}, a.tail...)))
}
target := targets[b]
full := append(target.tail[:len(target.tail):len(target.tail)], a.tail...)
if err := provePath(target.start, full, a.head, len(target.tail)); err != nil {
return a, err
}
w := compilePath(target.start, full)
a.kind, a.named, a.steps, a.leaves = namedRead, b, w.steps, w.leaves
a.levels = levelKeys(full, 0)
return a, nil
}
+44 -32
View File
@@ -6,17 +6,17 @@ import (
"testing" "testing"
) )
func benchGenerator(b *testing.B, dir string) *Generator { func benchGenerator(b *testing.B, data Option) *Generator {
b.Helper() b.Helper()
f, err := New(WithoutShippedData(), WithDataPath(dir), WithSeed(1)) f, err := New(data, WithSeed(1))
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
return f return f
} }
func benchPath(b *testing.B, dir, path string) { func benchPath(b *testing.B, data Option, path string) {
f := benchGenerator(b, dir) f := benchGenerator(b, data)
b.ReportAllocs() b.ReportAllocs()
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
@@ -26,12 +26,12 @@ func benchPath(b *testing.B, dir, path string) {
} }
} }
func BenchmarkPerson(b *testing.B) { benchPath(b, "data", "sv_SE.person") } func BenchmarkPerson(b *testing.B) { benchPath(b, withShipped(), "sv_SE.person") }
func BenchmarkAddress(b *testing.B) { benchPath(b, "data", "sv_SE.address") } func BenchmarkAddress(b *testing.B) { benchPath(b, withShipped(), "sv_SE.address") }
func BenchmarkWord(b *testing.B) { benchPath(b, "data", "sv_SE.word") } func BenchmarkWord(b *testing.B) { benchPath(b, withShipped(), "sv_SE.word") }
func BenchmarkCreditcard(b *testing.B) { benchPath(b, "data", "misc.creditcard") } func BenchmarkCreditcard(b *testing.B) { benchPath(b, withShipped(), "misc.creditcard") }
func BenchmarkPersonnummer(b *testing.B) { benchPath(b, "data", "sv_SE.personnummer") } func BenchmarkPersonnummer(b *testing.B) { benchPath(b, withShipped(), "sv_SE.personnummer") }
func BenchmarkUUIDv7(b *testing.B) { benchPath(b, "data", "misc.uuid") } func BenchmarkUUIDv7(b *testing.B) { benchPath(b, withShipped(), "misc.uuid") }
func tmpData(b *testing.B, name, body string) string { func tmpData(b *testing.B, name, body string) string {
b.Helper() b.Helper()
@@ -44,52 +44,64 @@ func tmpData(b *testing.B, name, body string) string {
func BenchmarkCalc(b *testing.B) { func BenchmarkCalc(b *testing.B) {
dir := tmpData(b, "inv", `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`) dir := tmpData(b, "inv", `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`)
benchPath(b, dir, "inv") benchPath(b, WithDataPath(dir), "inv")
} }
func BenchmarkLongLiteral(b *testing.B) { func BenchmarkLongLiteral(b *testing.B) {
dir := tmpData(b, "sql", `{"format":"INSERT INTO customers (id, name, city) VALUES (123, '{word}', '{word}');","word":["alpha","beta","gamma","delta"]}`) dir := tmpData(b, "sql", `{"format":"INSERT INTO customers (id, name, city) VALUES (123, '{word}', '{word}');","word":["alpha","beta","gamma","delta"]}`)
benchPath(b, dir, "sql") benchPath(b, WithDataPath(dir), "sql")
} }
func BenchmarkRepeat(b *testing.B) { func BenchmarkRepeat(b *testing.B) {
dir := tmpData(b, "many", `{"format":"{word}","repeat":20,"separator":", ","word":["alpha","beta","gamma","delta"]}`) dir := tmpData(b, "many", `{"format":"{word}","repeat":20,"separator":", ","word":["alpha","beta","gamma","delta"]}`)
benchPath(b, dir, "many") benchPath(b, WithDataPath(dir), "many")
} }
// BenchmarkBound measures a format that binds a head and reads two paths from it — // BenchmarkNamed measures a format that binds a field to a name and reads two paths through
// what a correlated pair costs against BenchmarkUnbound, the same output drawn from // it — what a correlated pair costs against BenchmarkUnnamed, the same output drawn from two
// two independent fields. // independent fields.
func BenchmarkBound(b *testing.B) { func BenchmarkNamed(b *testing.B) {
dir := tmpData(b, "addr", `{"format":"{place.postal-code} {place.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`) dir := tmpData(b, "addr", `{"format":"{place as p}{p.postal-code} {p.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`)
benchPath(b, dir, "addr") benchPath(b, WithDataPath(dir), "addr")
} }
func BenchmarkUnbound(b *testing.B) { func BenchmarkUnnamed(b *testing.B) {
dir := tmpData(b, "addr", `{"format":"{postal-code} {locality}","postal-code":["1{digits(2)} {digits(2)}","573 {digits(2)}"],"locality":["Stockholm","Tranås"]}`) dir := tmpData(b, "addr", `{"format":"{postal-code} {locality}","postal-code":["1{digits(2)} {digits(2)}","573 {digits(2)}"],"locality":["Stockholm","Tranås"]}`)
benchPath(b, dir, "addr") benchPath(b, WithDataPath(dir), "addr")
} }
// BenchmarkBoundDeep reads two paths through an intermediate level, the shape the // BenchmarkNamedDeep reads two paths through a name and an intermediate level.
// depth question is about. func BenchmarkNamedDeep(b *testing.B) {
func BenchmarkBoundDeep(b *testing.B) { dir := tmpData(b, "addr", `{"format":"{p as q}{q.addr.city} {q.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`)
dir := tmpData(b, "addr", `{"format":"{p.addr.city} {p.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`) benchPath(b, WithDataPath(dir), "addr")
benchPath(b, dir, "addr")
} }
// BenchmarkBoundWide reads ten paths from one row: the point where the maps an // BenchmarkNamedWide reads ten paths through one name: the point where the maps a pick keeps
// expansion holds outgrow a single bucket. // outgrow a single bucket.
func BenchmarkBoundWide(b *testing.B) { func BenchmarkNamedWide(b *testing.B) {
dir := tmpData(b, "row", `{"format":"{r.a}{r.b}{r.c}{r.d}{r.e}{r.f}{r.g}{r.h}{r.i}{r.j}","r":[ dir := tmpData(b, "row", `{"format":"{r as s}{s.a}{s.b}{s.c}{s.d}{s.e}{s.f}{s.g}{s.h}{s.i}{s.j}","r":[
{"format":"x","a":"1","b":"2","c":"3","d":"4","e":"5","f":"6","g":"7","h":"8","i":"9","j":"0"}, {"format":"x","a":"1","b":"2","c":"3","d":"4","e":"5","f":"6","g":"7","h":"8","i":"9","j":"0"},
{"format":"y","a":"A","b":"B","c":"C","d":"D","e":"E","f":"F","g":"G","h":"H","i":"I","j":"J"}]}`) {"format":"y","a":"A","b":"B","c":"C","d":"D","e":"E","f":"F","g":"G","h":"H","i":"I","j":"J"}]}`)
benchPath(b, dir, "row") benchPath(b, WithDataPath(dir), "row")
}
func BenchmarkFirstFake(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
f, err := New(withShipped(), WithSeed(1))
if err != nil {
b.Fatal(err)
}
if _, err := f.Fake("sv_SE.address"); err != nil {
b.Fatal(err)
}
}
} }
func BenchmarkNew(b *testing.B) { func BenchmarkNew(b *testing.B) {
b.ReportAllocs() b.ReportAllocs()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
if _, err := New(WithSeed(1)); err != nil { if _, err := New(withShipped(), WithSeed(1)); err != nil {
b.Fatal(err) b.Fatal(err)
} }
} }
+99 -51
View File
@@ -2,11 +2,14 @@ package fejkdata
import ( import (
"encoding/base64" "encoding/base64"
"fmt"
"regexp" "regexp"
"strconv" "strconv"
"strings" "strings"
"testing" "testing"
"time" "time"
"github.com/larvit/fejkdata/internal/invariant"
) )
func TestBuiltinIDGenerators(t *testing.T) { func TestBuiltinIDGenerators(t *testing.T) {
@@ -101,10 +104,10 @@ func TestBuiltinChecksums(t *testing.T) {
} }
} }
// TestBuiltinSeqPerSession pins seq's contract: a counter from 1, advancing on // TestBuiltinSeqPerGenerator pins seq's contract: a counter from 1, advancing on
// each call, named counters independent, and the whole thing scoped to one // each call, named counters independent, and the whole thing scoped to one
// session so a fresh Generator restarts at 1. // draw state so a fresh Generator restarts at 1.
func TestBuiltinSeqPerSession(t *testing.T) { func TestBuiltinSeqPerGenerator(t *testing.T) {
f := engine(1) f := engine(1)
for i := 1; i <= 5; i++ { for i := 1; i <= 5; i++ {
if got := mustRender(t, f, `"{seq()}"`); got != strconv.Itoa(i) { if got := mustRender(t, f, `"{seq()}"`); got != strconv.Itoa(i) {
@@ -121,9 +124,9 @@ func TestBuiltinSeqPerSession(t *testing.T) {
if got := mustRender(t, engine(2), `{"format":"{seq()}","repeat":3,"separator":","}`); got != "1,2,3" { if got := mustRender(t, engine(2), `{"format":"{seq()}","repeat":3,"separator":","}`); got != "1,2,3" {
t.Fatalf("repeat seq = %q, want 1,2,3", got) t.Fatalf("repeat seq = %q, want 1,2,3", got)
} }
// ...and a fresh session restarts from 1. // ...and a fresh generator restarts from 1.
if got := mustRender(t, engine(2), `"{seq()}"`); got != "1" { if got := mustRender(t, engine(2), `"{seq()}"`); got != "1" {
t.Fatalf("new session seq = %q, want 1", got) t.Fatalf("new generator seq = %q, want 1", got)
} }
} }
@@ -142,7 +145,7 @@ func TestBuiltinArgLimits(t *testing.T) {
`{"format":"{x}","x":"1","repeat":2000000000}`, // repeat -> multi-GB join `{"format":"{x}","x":"1","repeat":2000000000}`, // repeat -> multi-GB join
} }
for _, tmpl := range bad { for _, tmpl := range bad {
if _, err := compile(parse(t, tmpl)); err == nil { if _, err := resolved(t, tmpl); err == nil {
t.Errorf("compile(%s) = nil error, want a limit rejection", tmpl) t.Errorf("compile(%s) = nil error, want a limit rejection", tmpl)
} }
} }
@@ -187,51 +190,26 @@ func ibanValid(s string) bool {
return rem == 1 return rem == 1
} }
// TestRegistryShapes pins the builtin contract compileOps relies on: every entry // mustPanic fails unless call panics with invariant.Broken's phrase, which is what
// supplies prep, and args parsed at compile only behind a check. // separates a reported invariant break from a silently wrong value.
func TestRegistryShapes(t *testing.T) {
for name, b := range builtins {
if b.prep == nil {
t.Errorf("builtin %q has no prep: compileOps would call a nil func", name)
}
if b.arity != 0 && b.check == nil {
t.Errorf("builtin %q parses args in prep with no check", name)
}
}
}
// TestArgGuardsPanic pins the guards that report a builtin arg its check should
// have rejected. No data reaches them — checkFunc runs a builtin's check before
// compileOps ever calls prep — so they are exercised directly.
func TestArgGuardsPanic(t *testing.T) {
for name, call := range map[string]func(){
"atoi on an unvalidated arg": func() { atoi("nope") },
"atof on an unvalidated arg": func() { atof("nope") },
} {
mustPanic(t, name, call)
}
}
// mustPanic fails unless call panics, which is what separates a reported
// invariant break from a silently wrong value.
func mustPanic(t *testing.T, name string, call func()) { func mustPanic(t *testing.T, name string, call func()) {
t.Helper() t.Helper()
defer func() { defer func() {
if recover() == nil { if r := recover(); r == nil || !strings.HasPrefix(fmt.Sprint(r), invariant.Broken("")) {
t.Errorf("%s: no panic, want the invariant reported", name) t.Errorf("%s: recovered %v, want the invariant reported as an internal error", name, r)
} }
}() }()
call() call()
} }
func TestClassBuiltinArgs(t *testing.T) { func TestClassBuiltinArgs(t *testing.T) {
for _, bad := range []string{`"{digits(0)}"`, `"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} { for _, bad := range []string{`"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} {
if _, err := compile(parse(t, bad)); err == nil { if _, err := resolved(t, bad); err == nil {
t.Errorf("compile(%s) = nil error, want the arg rejected", bad) t.Errorf("compile(%s) = nil error, want the arg rejected", bad)
} }
} }
for _, ok := range []string{`"{digits(1048576)}"`, `"{upper(1)}"`, `"{lower(26)}"`} { for _, ok := range []string{`"{digits(1048576)}"`, `"{upper(1)}"`, `"{lower(26)}"`} {
if _, err := compile(parse(t, ok)); err != nil { if _, err := resolved(t, ok); err != nil {
t.Errorf("compile(%s) = %v", ok, err) t.Errorf("compile(%s) = %v", ok, err)
} }
} }
@@ -293,27 +271,22 @@ func TestBuiltinDateAndTime(t *testing.T) {
} }
} }
// TestBuiltinDateArgs pins the New-time checks: bounds are calendar dates in order, // TestBuiltinDateArgs pins the New-time checks: bounds are calendar dates in order, and the
// the layout is quoted, names a field, and for time names no date field. // layout is quoted.
func TestBuiltinDateArgs(t *testing.T) { func TestBuiltinDateArgs(t *testing.T) {
for tmpl, want := range map[string]string{ for tmpl, want := range map[string]string{
`"{date(1990-13-01,1990-12-31,'2006-01-02')}"`: "1990-13-01", `"{date(1990-13-01,1990-12-31,'2006-01-02')}"`: "1990-13-01",
`"{time('2006-01-02 15:04')}"`: "write date(from,to,layout)",
`"{date(1990-12-31,1990-01-01,'2006-01-02')}"`: "is after", `"{date(1990-12-31,1990-01-01,'2006-01-02')}"`: "is after",
`"{date(1990-01-01,1990-01-01,'2006-01-02')}"`: "write it as text",
`"{date(1990-01-01,1990-12-31,2006-01-02)}"`: "'2006-01-02'", `"{date(1990-01-01,1990-12-31,2006-01-02)}"`: "'2006-01-02'",
`"{date(1990-01-01,1990-12-31,'January 2, 2006)}"`: "'", `"{date(1990-01-01,1990-12-31,'January 2, 2006)}"`: "'",
`"{date(1990-01-01,1990-12-31,\"January 2, 2006\")}"`: "quoted: 'January 2, 2006'", `"{date(1990-01-01,1990-12-31,\"January 2, 2006\")}"`: "quoted: 'January 2, 2006'",
`"{date(1990-01-01,1990-12-31,'x')}"`: "text",
`"{date(1990-01-01,1990-12-31,'')}"`: "text",
`"{date(1990-01-01,1990-12-31)}"`: "3 arguments", `"{date(1990-01-01,1990-12-31)}"`: "3 arguments",
`"{date(1990-01-01,1990-12-31,January 2, 2006)}"`: "'January 2, 2006'", `"{date(1990-01-01,1990-12-31,January 2, 2006)}"`: "'January 2, 2006'",
`"{time(3:04 PM, Mon)}"`: "'3:04 PM, Mon'", `"{time(3:04 PM, Mon)}"`: "'3:04 PM, Mon'",
`"{time(15:04)}"`: "'15:04'", `"{time(15:04)}"`: "'15:04'",
`"{time('2006-01-02 15:04')}"`: "date(",
`"{time('x')}"`: "text",
`"{date(1990-01-01,1990-12-31,'15:04')}"`: "time('15:04')",
} { } {
_, err := compile(parse(t, tmpl)) _, err := resolved(t, tmpl)
if err == nil || !strings.Contains(err.Error(), want) { if err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("compile(%s) = %v, want an error mentioning %q", tmpl, err, want) t.Errorf("compile(%s) = %v, want an error mentioning %q", tmpl, err, want)
} }
@@ -349,7 +322,7 @@ func TestBuiltinLayoutErrorsNameARunnableSpelling(t *testing.T) {
} }
// TestBuiltinDateSpansOneDay pins that from == to is a day: with a clock layout it // TestBuiltinDateSpansOneDay pins that from == to is a day: with a clock layout it
// draws every second of it, and without one it could only emit one value. // draws every second of it, and without one it prints that day.
func TestBuiltinDateSpansOneDay(t *testing.T) { func TestBuiltinDateSpansOneDay(t *testing.T) {
f := engine(1) f := engine(1)
seen := map[string]bool{} seen := map[string]bool{}
@@ -363,8 +336,83 @@ func TestBuiltinDateSpansOneDay(t *testing.T) {
if len(seen) < 400 { if len(seen) < 400 {
t.Fatalf("date over one day drew %d distinct seconds in 500, want the whole day", len(seen)) t.Fatalf("date over one day drew %d distinct seconds in 500, want the whole day", len(seen))
} }
_, err := compile(parse(t, `"{date(2026-01-01,2026-01-01,'2006-01-02')}"`)) if got := mustRender(t, f, `"{date(2026-01-01,2026-01-01,'2006-01-02')}"`); got != "2026-01-01" {
if err == nil || !strings.Contains(err.Error(), "write it as text") { t.Fatalf("one day in a date-only layout = %q, want that day", got)
t.Fatalf("one day in a date-only layout = %v, want it named a constant", err) }
}
func TestClassBuiltins(t *testing.T) {
cases := map[string]*regexp.Regexp{
`"{digits(1)}"`: regexp.MustCompile(`^[0-9]$`),
`"{digits(3)}"`: regexp.MustCompile(`^[0-9]{3}$`),
`"{int(1,9)}"`: regexp.MustCompile(`^[1-9]$`),
`"{upper(1)}"`: regexp.MustCompile(`^[A-Z]$`),
`"{lower(1)}"`: regexp.MustCompile(`^[a-z]$`),
`"{upper(2)}{lower(2)}"`: regexp.MustCompile(`^[A-Z]{2}[a-z]{2}$`),
}
f := engine(7)
for tmpl, re := range cases {
for i := 0; i < 100; i++ {
if got := mustRender(t, f, tmpl); !re.MatchString(got) {
t.Fatalf("%s produced %q, want %s", tmpl, got, re)
}
}
}
}
func TestFunctionTokenLuhn(t *testing.T) {
// {luhn()} appends a Luhn check digit over the digits emitted so far in the
// current expansion (non-digits skipped but kept). It reads the output
// buffer, so a value is never re-rendered. Bodies are escaped to fix input.
f := engine(1)
cases := map[string]string{
`"811218987{luhn()}"`: "8112189876", // personnummer body
`"7992739871{luhn()}"`: "79927398713", // classic Luhn vector
`"811218-987{luhn()}"`: "811218-9876", // '-' skipped, kept
`{"format":"{n}{luhn()}","n":"811218987"}`: "8112189876", // over a rendered token
}
for tmpl, want := range cases {
if got := mustRender(t, f, tmpl); got != want {
t.Fatalf("%s = %q, want %q", tmpl, got, want)
}
}
}
func TestBuiltinCompileErrors(t *testing.T) {
for _, bad := range []string{
`"{upper(x)}"`, // count must be an integer
`"{int(a,b)}"`, // non-integer args
`"{int(5,1)}"`, // min > max
`"{nanoid(-1)}"`, // negative count
`"{float(1,2,-1)}"`, // negative decimals
`"{float(NaN,NaN,2)}"`, // bounds must be finite
`"{float(Inf,Inf,2)}"`, // same-sign infinities
`"{float(1,NaN,2)}"`, // one NaN bound
`"{float(-Inf,1,2)}"`, // one infinite bound
`"{iban(US)}"`, // unsupported country
`"{seq(a,b)}"`, // seq takes at most one name
} {
if _, err := resolved(t, bad); err == nil {
t.Errorf("compile(%s) = nil error, want error", bad)
}
}
}
func TestArgErrorsNameTheSpelling(t *testing.T) {
if _, err := resolved(t, `"{hex(99999999999999999999)}"`); err == nil || !strings.Contains(err.Error(), "exceeds the maximum") {
t.Errorf("compile(hex past the range) = %v, want the maximum named", err)
}
}
func TestASignOrLeadingZeroIsTheInteger(t *testing.T) {
for long, short := range map[string]string{
`"{float(1,2,02)}"`: `"{float(1,2,2)}"`,
`{"format":"{calc(a,02)}","a":"1"}`: `{"format":"{calc(a,2)}","a":"1"}`,
`"{digits(+5)}"`: `"{digits(5)}"`,
`"{digits(05)}"`: `"{digits(5)}"`,
`"{int(007,9)}"`: `"{int(7,9)}"`,
`"{int(+1,5)}"`: `"{int(1,5)}"`,
} {
sameRenders(t, long, short)
} }
} }
+93 -336
View File
@@ -2,392 +2,149 @@ package fejkdata
import ( import (
"fmt" "fmt"
"math" "slices"
"strconv" "strconv"
"strings" "strings"
"unicode"
"github.com/larvit/fejkdata/internal/builtinfunc"
"github.com/larvit/fejkdata/internal/grammar"
"github.com/larvit/fejkdata/internal/invariant"
) )
// calcNode is a parsed expression node. It evaluates over the operand values expand // checkCalcFields runs checkOperands over each calc operand naming a field; an operand naming no
// read, so the evaluator touches neither the rng nor the node tree. // field is a name, which checkCalcNames checks once names resolve.
type calcNode interface { func checkCalcFields(args []string, fields map[string]node) error {
eval(operands []string) float64 return checkOperands(args[0], builtinfunc.ParsedCalc(args[0]), func(name string) []node {
} if n, ok := fields[name]; ok {
return []node{n}
type calcNum float64 // a number literal
type calcVar string // a sibling-field name, before indexVars places it
type calcIdx int // an operand, by its position in the values expand read
type calcNeg struct{ x calcNode }
type calcBin struct { // a + - * / b
op byte
l, r calcNode
}
func (n calcNum) eval([]string) float64 { return float64(n) }
func (n calcIdx) eval(operands []string) float64 {
v, err := strconv.ParseFloat(strings.TrimSpace(operands[n]), 64)
if err != nil {
return math.NaN() // a non-numeric operand stays visible, never an error
}
return v
}
// eval on an unplaced name cannot happen: calcPrep runs indexVars over every
// expression it compiles, so only a calcIdx reaches a render. It panics rather
// than returning NaN, so a node kind indexVars forgets is a stack trace and not a
// silently wrong number.
func (n calcVar) eval([]string) float64 {
panic(fmt.Sprintf("fejkdata: calc operand %q was never placed", string(n)))
}
func (n calcNeg) eval(operands []string) float64 { return -n.x.eval(operands) }
func (n calcBin) eval(operands []string) float64 {
l, r := n.l.eval(operands), n.r.eval(operands)
switch n.op {
case '+':
return l + r
case '-':
return l - r
case '*':
return l * r
default: // '/'
return l / r
}
}
// checkCalc validates a calc token at compile time: a parseable expression whose
// operands all name existing fields, and an optional non-negative integer dp. It
// is a builtin check (fields first), so calc dispatches through the registry like
// every other {name(args)} function.
func checkCalc(fields map[string]node, args []string) error {
if len(args) < 1 || len(args) > 2 {
return fmt.Errorf("calc takes an expression and an optional decimals count, got %d args", len(args))
}
expr, err := parseCalc(args[0])
if err != nil {
return fmt.Errorf("calc(%q): %w", args[0], err)
}
for _, name := range calcVars(expr) {
operand, ok := fields[name]
if !ok {
return fmt.Errorf("calc(%q): no field %q", args[0], name)
} }
if text, never := neverNumeric(operand); never { return nil
return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", args[0], name, text) })
}
// checkCalcNames holds each calc of t reading a name to the checks checkCalcFields makes of a field.
func checkCalcNames(label string, t *template) error {
for _, o := range t.compiled.ops {
if o.Fn != builtinfunc.CalcName || !slices.ContainsFunc(o.operands, func(a arm) bool { return a.kind == namedRead }) {
continue
}
if err := checkOperands(o.Args[0], builtinfunc.ParsedCalc(o.Args[0]), operandNodes(o)); err != nil {
return fmt.Errorf("%s: token {%s}: %w", label, o.Body, err)
} }
} }
if divisor, zero := constantZeroDivisor(expr, fields); zero { return nil
return fmt.Errorf("calc(%q) divides by %s, which is always zero", args[0], divisor) }
// operandNodes lists every node each operand of o may render, once its template and names resolve.
func operandNodes(o op) func(name string) []node {
return func(name string) []node {
a := o.operands[slices.IndexFunc(o.operands, func(a arm) bool { return a.head == name })]
if len(a.leaves) == 0 {
panic(invariant.Broken("calc operand %q was read before it was compiled", name))
}
return a.leaves
} }
if len(args) == 2 { }
dp, err := plainInt(args[1])
if err != nil { // checkOperands refuses an operand that is never a number and a division by a constant zero.
return fmt.Errorf("calc decimals %w", err) // operand lists every node an operand may render, nil while that is unknown.
} func checkOperands(text string, c grammar.Calc, operand func(name string) []node) error {
if dp < 0 || dp > maxDecimals { for _, name := range c.Operands {
return fmt.Errorf("calc decimals %d must be in 0..%d", dp, maxDecimals) if rendered, never := allNeverNumeric(operand(name)); never {
return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", text, name, rendered)
} }
} }
if divisor, zero := constantZeroDivisor(c.Expr, operand); zero {
return fmt.Errorf("calc(%q) divides by %s, which is always zero", text, divisor)
}
return nil return nil
} }
// constantZeroDivisor finds a division whose right side is a constant zero: number // constantZeroDivisor finds a division whose right side is a constant zero: number
// literals and fixed operands folded, anything that varies left unknown. // literals and fixed operands folded, anything that varies left unknown.
func constantZeroDivisor(n calcNode, fields map[string]node) (string, bool) { func constantZeroDivisor(n grammar.CalcNode, operand func(string) []node) (string, bool) {
switch n := n.(type) { switch n := n.(type) {
case calcNeg: case grammar.CalcNeg:
return constantZeroDivisor(n.x, fields) return constantZeroDivisor(n.X, operand)
case calcBin: case grammar.CalcBin:
if n.op == '/' { if n.Operator == '/' {
if v, known := constantValue(n.r, fields); known && v == 0 { if v, known := constantValue(n.R, operand); known && v == 0 {
return calcText(n.r), true return grammar.CalcText(n.R), true
} }
} }
if d, zero := constantZeroDivisor(n.l, fields); zero { if d, zero := constantZeroDivisor(n.L, operand); zero {
return d, true return d, true
} }
return constantZeroDivisor(n.r, fields) return constantZeroDivisor(n.R, operand)
} }
return "", false return "", false
} }
// constantValue evaluates an expression whose every operand is fixed. // constantValue evaluates an expression whose every operand is fixed.
func constantValue(n calcNode, fields map[string]node) (float64, bool) { func constantValue(n grammar.CalcNode, operand func(string) []node) (float64, bool) {
switch n := n.(type) { switch n := n.(type) {
case calcNum: case grammar.CalcNum:
return float64(n), true return float64(n), true
case calcVar: case grammar.CalcVar:
t, ok := fields[string(n)].(*template) nodes := operand(n.Name)
if !ok || !t.fixed || t.repeat > 1 { if len(nodes) != 1 {
return 0, false return 0, false
} }
v, err := strconv.ParseFloat(strings.TrimSpace(t.lit), 64) t, ok := nodes[0].(*template)
if !ok || t.repeat != 1 {
return 0, false
}
lit, fixed := t.fixedText()
if !fixed {
return 0, false
}
v, err := strconv.ParseFloat(strings.TrimSpace(lit), 64)
return v, err == nil return v, err == nil
case calcNeg: case grammar.CalcNeg:
v, ok := constantValue(n.x, fields) v, ok := constantValue(n.X, operand)
return -v, ok return -v, ok
case calcBin: case grammar.CalcBin:
l, lok := constantValue(n.l, fields) l, lok := constantValue(n.L, operand)
r, rok := constantValue(n.r, fields) r, rok := constantValue(n.R, operand)
if !lok || !rok { if !lok || !rok {
return 0, false return 0, false
} }
return calcBin{n.op, calcNum(l), calcNum(r)}.eval(nil), true return builtinfunc.Arith(n.Operator, l, r), true
} }
return 0, false return 0, false
} }
// calcText spells an expression node the way an author would read it. // neverNumeric reports a node no render of which is a number: a null, a repeat of 0, fixed
func calcText(n calcNode) string { // text that does not parse, or a choice of only such items. text is one such render.
switch n := n.(type) {
case calcNum:
return strconv.FormatFloat(float64(n), 'f', -1, 64)
case calcVar:
return string(n)
case calcNeg:
return "-" + calcText(n.x)
case calcBin:
return "(" + calcText(n.l) + " " + string(n.op) + " " + calcText(n.r) + ")"
}
return "?"
}
// neverNumeric reports a node no render of which is a number: a null, fixed text that
// does not parse, or a choice of only such items. text is one such render.
func neverNumeric(n node) (text string, never bool) { func neverNumeric(n node) (text string, never bool) {
switch n := n.(type) { switch n := n.(type) {
case *null: case *nullItem:
return "", true return "", true
case *template: case *template:
if !n.fixed || n.repeat > 1 { lit, fixed := n.fixedText()
if n.repeat == 0 {
return "", true
}
if !fixed || n.repeat != 1 {
return "", false return "", false
} }
if _, err := strconv.ParseFloat(strings.TrimSpace(n.lit), 64); err != nil { if _, err := strconv.ParseFloat(strings.TrimSpace(lit), 64); err != nil {
return n.lit, true return lit, true
} }
case *choice: case *choice:
for _, it := range n.items { return allNeverNumeric(n.items)
t, itemNever := neverNumeric(it)
if !itemNever {
return "", false
}
text = t
}
return text, true
} }
return "", false return "", false
} }
// calcPrep parses the expression and decimals once, at compile time, and places each // allNeverNumeric reports whether no render of any of nodes is a number; false for no nodes.
// operand name at the position expand will read it into. checkCalc proved both args func allNeverNumeric(nodes []node) (text string, never bool) {
// valid, so no step here can fail; dp -1 prints the minimal form. for _, n := range nodes {
func calcPrep(args []string) callFn { t, nodeNever := neverNumeric(n)
expr, err := parseCalc(args[0]) if !nodeNever {
if err != nil { // a nil AST would be a nil dereference per render, with no message return "", false
panic(fmt.Sprintf("fejkdata: calc(%q) reached prep unparsed: %v", args[0], err))
}
at := make(map[string]int)
for i, name := range calcVars(expr) {
at[name] = i
}
placed := indexVars(expr, at)
dp := calcDecimals(args)
return func(_ *session, _ string, operands []string) string {
return formatFloat(placed.eval(operands), dp)
}
}
// calcDecimals is a calc's decimals count, or -1 for the shortest form.
func calcDecimals(args []string) int {
if len(args) == 2 {
return atoi(args[1])
}
return -1
}
// indexVars replaces each operand name with its position in the values expand reads.
// Both sides take that order from calcVars, so they cannot drift.
func indexVars(n calcNode, at map[string]int) calcNode {
switch n := n.(type) {
case calcVar:
i, placed := at[string(n)]
if !placed { // calcVars named every operand, so a miss means the two disagree
panic(fmt.Sprintf("fejkdata: calc operand %q is not among the names read for it", string(n)))
} }
return calcIdx(i) text = t
case calcNeg:
return calcNeg{indexVars(n.x, at)}
case calcBin:
return calcBin{n.op, indexVars(n.l, at), indexVars(n.r, at)}
} }
return n return text, len(nodes) > 0
}
// calcOperands lists the sibling-field names a calc's args read. checkCalc reports
// an expression that does not parse, so one that does not simply names nothing.
func calcOperands(args []string) []string {
if len(args) == 0 {
return nil
}
expr, err := parseCalc(args[0])
if err != nil {
return nil
}
return calcVars(expr)
}
// calcVars lists the distinct field names an expression reads, in the order it first
// names each. That order is the contract between expand, which reads the operands
// into a slice, and indexVars, which places each name at its position in it.
func calcVars(n calcNode) []string {
var out []string
seen := map[string]bool{}
var walk func(calcNode)
walk = func(n calcNode) {
switch n := n.(type) {
case calcVar:
if name := string(n); !seen[name] {
seen[name] = true
out = append(out, name)
}
case calcNeg:
walk(n.x)
case calcBin:
walk(n.l)
walk(n.r)
}
}
walk(n)
return out
}
// calcParser is a recursive-descent parser over the expression runes, threading
// expr -> term -> factor for the standard * / before + - precedence.
type calcParser struct {
rs []rune
pos int
}
// parseCalc parses a whole expression, requiring it to consume all input.
func parseCalc(expr string) (calcNode, error) {
p := &calcParser{rs: []rune(expr)}
if p.space(); p.pos >= len(p.rs) {
return nil, fmt.Errorf("empty expression")
}
n, err := p.expr()
if err != nil {
return nil, err
}
if p.space(); p.pos != len(p.rs) {
return nil, fmt.Errorf("unexpected %q", string(p.rs[p.pos:]))
}
return n, nil
}
func (p *calcParser) space() {
for p.pos < len(p.rs) && unicode.IsSpace(p.rs[p.pos]) {
p.pos++
}
}
func (p *calcParser) expr() (calcNode, error) { return p.binary(p.term, '+', '-') }
func (p *calcParser) term() (calcNode, error) { return p.binary(p.factor, '*', '/') }
// binary parses a left-associative run of next() operands joined by the given
// operators, the one shape expr and term share.
func (p *calcParser) binary(next func() (calcNode, error), ops ...byte) (calcNode, error) {
n, err := next()
if err != nil {
return nil, err
}
for {
p.space()
if p.pos >= len(p.rs) || !contains(ops, byte(p.rs[p.pos])) {
return n, nil
}
op := byte(p.rs[p.pos])
p.pos++
r, err := next()
if err != nil {
return nil, err
}
n = calcBin{op, n, r}
}
}
// factor is a table-shaped scanner, one case per token kind, kept whole on purpose.
func (p *calcParser) factor() (calcNode, error) {
p.space()
if p.pos >= len(p.rs) {
return nil, fmt.Errorf("unexpected end of expression")
}
switch c := p.rs[p.pos]; {
case c == '-':
p.pos++
x, err := p.factor()
if err != nil {
return nil, err
}
return calcNeg{x}, nil
case c == '(':
p.pos++
n, err := p.expr()
if err != nil {
return nil, err
}
if p.space(); p.pos >= len(p.rs) || p.rs[p.pos] != ')' {
return nil, fmt.Errorf("missing ')'")
}
p.pos++
return n, nil
case c == '.' || c >= '0' && c <= '9':
return p.number()
case c == '_' || unicode.IsLetter(c):
return p.ident()
default:
return nil, fmt.Errorf("unexpected %q", string(c))
}
}
func (p *calcParser) number() (calcNode, error) {
start, dot := p.pos, false
for p.pos < len(p.rs) {
if c := p.rs[p.pos]; c >= '0' && c <= '9' {
p.pos++
} else if c == '.' && !dot {
dot, p.pos = true, p.pos+1
} else {
break
}
}
v, err := strconv.ParseFloat(string(p.rs[start:p.pos]), 64)
if err != nil {
return nil, fmt.Errorf("bad number %q", string(p.rs[start:p.pos]))
}
return calcNum(v), nil
}
// ident reads a field name: a letter or '_', then letters, digits or '_'. A '-'
// is always the minus operator, so a hyphenated field name can't be an operand.
func (p *calcParser) ident() (calcNode, error) {
start := p.pos
for p.pos < len(p.rs) {
if c := p.rs[p.pos]; c == '_' || unicode.IsLetter(c) || unicode.IsDigit(c) {
p.pos++
} else {
break
}
}
return calcVar(string(p.rs[start:p.pos])), nil
}
func contains(bs []byte, b byte) bool {
for _, x := range bs {
if x == b {
return true
}
}
return false
} }
+140 -64
View File
@@ -3,8 +3,11 @@ package fejkdata
import ( import (
"fmt" "fmt"
"math" "math"
"strconv"
"strings" "strings"
"testing" "testing"
"github.com/larvit/fejkdata/internal/grammar"
) )
// TestCalcArithmetic pins the operators, precedence, parentheses and unary minus // TestCalcArithmetic pins the operators, precedence, parentheses and unary minus
@@ -72,6 +75,9 @@ func TestCalcFields(t *testing.T) {
if got := mustRender(t, engine(1), `{"format":"{calc(a - b)}","a":{"format":"{n}","n":"10"},"b":"3"}`); got != "7" { if got := mustRender(t, engine(1), `{"format":"{calc(a - b)}","a":{"format":"{n}","n":"10"},"b":"3"}`); got != "7" {
t.Fatalf("calc(a - b) = %q, want 7", got) t.Fatalf("calc(a - b) = %q, want 7", got)
} }
if got := mustRender(t, engine(1), `{"format":"{calc(b * (a - b))}","a":"10","b":"3"}`); got != "21" {
t.Fatalf("calc(b * (a - b)) = %q, want 21", got)
}
} }
// TestCalcNonNumericIsNaN pins the never-fail rule: a field that sometimes does // TestCalcNonNumericIsNaN pins the never-fail rule: a field that sometimes does
@@ -94,43 +100,26 @@ func TestCalcReproducible(t *testing.T) {
} }
} }
// TestTokenOperandsReadsOnlyAnOperandBuiltin pins what the helper answers for a body that // TestParsedCallReadsOnlyAnOperandBuiltin pins the operands a parsed call carries for a
// is not a calc, and for one whose expression does not parse: nothing, either way. // call that is not a calc, and for one whose expression does not parse: none, either way.
// checkCalc is what reports a bad expression, so the callers that run after it func TestParsedCallReadsOnlyAnOperandBuiltin(t *testing.T) {
// never meet one — but they must not have to depend on that order to be safe. for _, format := range []string{"{luhn()}", "{calc()}", "{calc(1 +)}", "{calc(()}"} {
func TestTokenOperandsReadsOnlyAnOperandBuiltin(t *testing.T) { if toks, err := grammar.ParseFormat(format); err != nil || len(toks) != 1 || tokenReads(toks[0]) != nil {
for _, body := range []string{"plain", "luhn()", "calc()", "calc(1 +)", "calc(()"} { t.Errorf("ParseFormat(%q) = %+v, %v, want one call reading no operand", format, toks, err)
if got := tokenOperands(body); got != nil {
t.Errorf("tokenOperands(%q) = %v, want none", body, got)
} }
} }
if got := tokenOperands("calc(net * qty)"); len(got) != 2 { if toks, err := grammar.ParseFormat("{calc(net * qty)}"); err != nil || len(toks) != 1 || len(tokenReads(toks[0])) != 2 {
t.Errorf("tokenOperands(calc(net * qty)) = %v, want both operands", got) t.Errorf("ParseFormat({calc(net * qty)}) = %+v, %v, want both operands", toks, err)
}
}
// TestCalcGuardsPanic pins the guards on calc's own invariants: checkCalc parsed
// the expression before prep sees it, and indexVars places every name calcVars
// read, so a break in either is reported rather than computed around.
func TestCalcGuardsPanic(t *testing.T) {
for name, call := range map[string]func(){
"prep on an expression that does not parse": func() { calcPrep([]string{"1 +"}) },
"an operand name never placed": func() { calcVar("n").eval(nil) },
"indexVars on a name it did not read": func() { indexVars(calcVar("n"), map[string]int{}) },
} {
mustPanic(t, name, call)
} }
} }
// --- one draw, one value: a calc operand reads the expansion's draw --- // --- one draw, one value: a calc operand reads the expansion's draw ---
// TestCalcOperandReadsTheExpansionsDraw pins the one-draw rule for a calc operand. // TestANamedCalcOperandIsTheOneShown pins how a calc computes from the value its format
// A field the format renders and a calc reads is drawn once per expansion, so the // shows: the format and the calc read one name.
// operand shown is the operand computed — the correlation the dotted-path rule func TestANamedCalcOperandIsTheOneShown(t *testing.T) {
// already gives a level (see bound_test.go), applied to a plain sibling.
func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) {
dir := writeData(t, map[string]string{ dir := writeData(t, map[string]string{
"inv": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`, "inv": `{"format":"{net as n}{qty as q}{n} x {q} = {calc(n * q, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
}) })
f := newGenerator(t, dir, WithSeed(3)) f := newGenerator(t, dir, WithSeed(3))
for i := 0; i < 300; i++ { for i := 0; i < 300; i++ {
@@ -145,39 +134,23 @@ func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) {
} }
} }
// TestCalcOperandSharesOneDraw pins the reach of that hold: the draw belongs to the // TestACalcOperandDrawsAfresh pins that a calc reading a field draws it as any {…} does.
// expansion, not to the calc, so the bare token rendering the same name reads it too. func TestACalcOperandDrawsAfresh(t *testing.T) {
func TestCalcOperandSharesOneDraw(t *testing.T) { dir := writeData(t, map[string]string{"same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`})
dir := writeData(t, map[string]string{
"same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`,
})
f := newGenerator(t, dir, WithSeed(5)) f := newGenerator(t, dir, WithSeed(5))
for i := 0; i < 200; i++ { for i := 0; i < 200; i++ {
got := fake(t, f, "same") if p := strings.Fields(fake(t, f, "same")); p[0] != p[1] {
if p := strings.Fields(got); len(p) != 2 || p[0] != p[1] {
t.Fatalf("same = %q, want one value twice", got)
}
}
}
// TestFieldNoCalcReadsDrawsEachTime guards the boundary: only a name a calc reads is
// held, so an ordinary {w} {w} still draws twice.
func TestFieldNoCalcReadsDrawsEachTime(t *testing.T) {
dir := writeData(t, map[string]string{"two": `{"format":"{w} {w}","w":["1","2","3","4","5"]}`})
f := newGenerator(t, dir, WithSeed(5))
for i := 0; i < 200; i++ {
if p := strings.Fields(fake(t, f, "two")); p[0] != p[1] {
return return
} }
} }
t.Fatal("{w} {w} never differed in 200 draws, want two independent draws") t.Fatal("{w} {calc(w)} never differed in 200 draws, want two independent draws")
} }
// TestCalcHoldIsPerExpansion pins the scope of the hold: each repeat iteration is // TestANamedCalcOperandPicksEachIteration pins that a name a repeat binds picks again on each
// its own expansion, so it draws again while staying self-consistent. // iteration, staying one value within it.
func TestCalcHoldIsPerExpansion(t *testing.T) { func TestANamedCalcOperandPicksEachIteration(t *testing.T) {
dir := writeData(t, map[string]string{ dir := writeData(t, map[string]string{
"rep": `{"format":"{n}={calc(n * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`, "rep": `{"format":"{n as m}{m}={calc(m * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`,
}) })
f := newGenerator(t, dir, WithSeed(11)) f := newGenerator(t, dir, WithSeed(11))
varied := false varied := false
@@ -198,12 +171,12 @@ func TestCalcHoldIsPerExpansion(t *testing.T) {
} }
} }
// TestCalcHoldIsPerTemplate pins that a nested template holds its own: the inner // TestTwoNamesPickApart pins that a nested template's name is a pick of its own: the inner
// {v} and its calc agree with each other, not with the outer pair. // pair agrees with itself, not with the outer pair.
func TestCalcHoldIsPerTemplate(t *testing.T) { func TestTwoNamesPickApart(t *testing.T) {
dir := writeData(t, map[string]string{ dir := writeData(t, map[string]string{
"nest": `{"format":"{v}={calc(v * 1)} {inner}","v":["2","3","4","5","6","7","8","9"], "nest": `{"format":"{v as a}{a}={calc(a * 1)} {inner}","v":["2","3","4","5","6","7","8","9"],
"inner":{"format":"{v}={calc(v * 1)}","v":["2","3","4","5","6","7","8","9"]}}`, "inner":{"format":"{v as b}{b}={calc(b * 1)}","v":["2","3","4","5","6","7","8","9"]}}`,
}) })
f := newGenerator(t, dir, WithSeed(13)) f := newGenerator(t, dir, WithSeed(13))
differed := false differed := false
@@ -231,7 +204,7 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
`{"format":"{calc(x * 2)}","x":["a","b"]}`: `"x"`, `{"format":"{calc(x * 2)}","x":["a","b"]}`: `"x"`,
`{"format":"{calc(x + y)}","x":"1","y":"{{2}}"}`: `"{2}"`, `{"format":"{calc(x + y)}","x":"1","y":"{{2}}"}`: `"{2}"`,
} { } {
_, err := compile(parse(t, src)) _, err := resolved(t, src)
if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "never a number") { if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "never a number") {
t.Errorf("compile(%s) = %v, want the operand rejected naming %s", src, err, want) t.Errorf("compile(%s) = %v, want the operand rejected naming %s", src, err, want)
} }
@@ -242,7 +215,7 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
`{"format":"{calc(x * 2)}","x":["1","abc"]}`, `{"format":"{calc(x * 2)}","x":["1","abc"]}`,
`{"format":"{calc(x * 2)}","x":"{digits(2)}"}`, `{"format":"{calc(x * 2)}","x":"{digits(2)}"}`,
} { } {
if _, err := compile(parse(t, ok)); err != nil { if _, err := resolved(t, ok); err != nil {
t.Errorf("compile(%s) = %v, want it accepted", ok, err) t.Errorf("compile(%s) = %v, want it accepted", ok, err)
} }
} }
@@ -250,14 +223,14 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
func TestCalcConstantZeroDivisorIsRejected(t *testing.T) { func TestCalcConstantZeroDivisorIsRejected(t *testing.T) {
for src, want := range map[string]string{ for src, want := range map[string]string{
`"{calc(1/0)}"`: "divides by 0", `"{calc(1/0)}"`: "divides by 0",
`"{calc(2/(1-1))}"`: "divides by (1 - 1)", `"{calc(2/(1-1))}"`: "divides by (1 - 1)",
`{"format":"{calc(x/y)}","x":"1","y":"0"}`: "divides by y", `{"format":"{calc(x/y)}","x":"1","y":"0"}`: "divides by y",
`{"format":"{calc(x/(y*2))}","x":"1","y":" 0 "}`: "divides by (y * 2)", `{"format":"{calc(x/(y*2))}","x":"1","y":" 0 "}`: "divides by (y * 2)",
`{"format":"{calc((a/0)+b)}","a":"1","b":"2"}`: "divides by 0", `{"format":"{calc((a/0)+b)}","a":"1","b":"2"}`: "divides by 0",
`{"format":"{calc(a/-0)}","a":"1"}`: "divides by -0", `{"format":"{calc(a/-0)}","a":"1"}`: "divides by -0",
} { } {
if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) { if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("compile(%s) = %v, want the constant zero divisor rejected naming %q", src, err, want) t.Errorf("compile(%s) = %v, want the constant zero divisor rejected naming %q", src, err, want)
} }
} }
@@ -272,3 +245,106 @@ func TestCalcConstantZeroDivisorIsRejected(t *testing.T) {
} }
t.Fatal("a sometimes-zero divisor never printed +Inf in 50 draws") t.Fatal("a sometimes-zero divisor never printed +Inf in 50 draws")
} }
func TestCalcCompileErrors(t *testing.T) {
for _, bad := range []string{
`"{calc(1/0)}"`, // a constant zero divisor
`{"format":"{calc(x/y)}","x":"1","y":"0"}`, // a fixed zero divisor
`"{calc()}"`, // calc needs an expression
`"{calc(1 +)}"`, // dangling operator
`"{calc((1 + 2)}"`, // unbalanced parenthesis
`"{calc(1 2)}"`, // two operands, no operator
`"{calc(price)}"`, // operand names no field
`"{calc(1, 2, 3)}"`, // too many args
`"{calc(1, x)}"`, // decimals arg not an integer
`"{calc(1, -1)}"`, // decimals negative
} {
if _, err := resolved(t, bad); err == nil {
t.Errorf("compile(%s) = nil error, want error", bad)
}
}
}
func TestCalcReadsAName(t *testing.T) {
dir := writeData(t, map[string]string{
"n": `["2","3","5","7"]`,
"card": `"{/n as k}{k} x 2 = {calc(k * 2)}"`,
})
f := newGenerator(t, dir, WithSeed(1))
for i := 0; i < 50; i++ {
var k, double int
got := fake(t, f, "card")
if _, err := fmt.Sscanf(got, "%d x 2 = %d", &k, &double); err != nil || double != k*2 {
t.Fatalf("card = %q, want the calc to compute from the pick {k} prints", got)
}
}
}
func TestCalcReadsANameItReadsNowhereElse(t *testing.T) {
dir := writeData(t, map[string]string{
"n": `["2","3"]`,
"card": `"{/n as k}{calc(k * 2)}"`,
})
f := newGenerator(t, dir, WithSeed(1))
if got := fake(t, f, "card"); got != "4" && got != "6" {
t.Fatalf("card = %q, want 4 or 6", got)
}
}
func TestCalcRefusesABadOperand(t *testing.T) {
for card, want := range map[string]string{
`"{/word as w}{w}{calc(w * 2)}"`: `operand "w" is never a number: it renders "def"`,
`"{/zero as z}{z}{calc(1 / z)}"`: `divides by z, which is always zero`,
`"{calc(nope * 2)}"`: `no field "nope"`,
`"{/word as w}{w}{calc(nope * 2)}"`: `no field or name "nope"; the names bound here are "w"`,
} {
dir := writeData(t, map[string]string{
"word": `["abc","def"]`,
"zero": `"0"`,
"card": card,
})
if _, err := New(withShipped(), WithDataPath(dir), WithSeed(1)); err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("New with card %s = %v, want an error containing %q", card, err, want)
}
}
}
func TestATypedColumnRefusesACalcOverNoDigits(t *testing.T) {
_, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{
"row": `{"format":"","x":{"format":"{calc(d + 1)}","d":"{digits(0)}","datatype":"integer"}}`,
})))
if err == nil || !strings.Contains(err.Error(), "{digits(0)} prints nothing, which is no number") {
t.Fatalf("New = %v, want a calc over {digits(0)} refused in a typed column", err)
}
}
func TestATypedColumnProvesACalcOverAName(t *testing.T) {
dir := writeData(t, map[string]string{
"n": `"{int(1,9)}"`,
"order": `{"format":"{/n as k}{k}","double":{"format":"{calc(k * 2)}","datatype":"integer"}}`,
})
f := newGenerator(t, dir, WithSeed(1))
r, err := f.FakeRecord("order")
if err != nil {
t.Fatal(err)
}
if v, err := strconv.Atoi(r.Columns()[0].Value); err != nil || v < 2 || v > 18 || v%2 != 0 {
t.Fatalf("double = %q, want an even integer in 2..18", r.Columns()[0].Value)
}
if !strings.Contains(r.JSON(), `"double":`+r.Columns()[0].Value) {
t.Fatalf("JSON = %s, want double written as a number", r.JSON())
}
bad := writeData(t, map[string]string{
"word": `["1","x"]`,
"order": `{"format":"{/word as w}{w}","double":{"format":"{calc(w * 2)}","datatype":"integer"}}`,
})
if _, err := New(withShipped(), WithDataPath(bad), WithSeed(1)); err == nil || !strings.Contains(err.Error(), `operand "w": "x" is not a number`) {
t.Fatalf("New = %v, want the typed column refused over an operand that is not always a number", err)
}
}
func TestARepeatOfZeroIsNeverANumber(t *testing.T) {
if _, err := resolved(t, `{"format":"{calc(x / y)}","x":"1","y":{"format":"0","repeat":0}}`); err == nil || !strings.Contains(err.Error(), `operand "y" is never a number: it renders ""`) {
t.Errorf("a calc over a repeat of 0 = %v, want the operand refused as never a number", err)
}
}
+706
View File
@@ -0,0 +1,706 @@
// Package cli is the fejkdata command: it renders the template on its stdin with fake
// values from the modules it is given and any directories layered over them. A program
// passing its own modules to Main is a fejkdata CLI carrying them, for example
// os.Exit(cli.Main(append(data.Modules(), mydata)...)).
package cli
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"math/rand/v2"
"os"
"path/filepath"
"reflect"
"runtime/debug"
"sort"
"strconv"
"strings"
"github.com/larvit/fejkdata"
)
const usage = `Usage: <template> | COMMAND [flags]
COMMAND --list | --help | --version
COMMAND reads one template from stdin and renders it:
echo 'id: {int(1,999)}' | COMMAND
echo '{"format":"{x}","x":["bosse","lina"]}' | COMMAND
COMMAND < template.txt
COMMAND <<'EOF'
born {date(1990-01-01,2010-12-31,'2006-01-02')}
EOF
A template is a format string, or a JSON object, array, string or null. Its {…} tokens
reach the data by reference from the root: {/path}, for any path --list prints, shipped
or layered with --data-path. A reference names a category or a dotted path into one, a
table's row by key or name ({/table[KEY]}), and .. steps up from a row to the row it
links to.
Under --format text, the default, what you send is what renders, and COMMAND adds
nothing: a format string keeps every byte, so echo's newline ends each render, and
printf '%s' sends none. -n joins renders with --separator, empty by default. A JSON
object, array or string drops the whitespace around it, and null alone renders nothing;
42 or true alone is a format string, printed as written. An empty or blank stdin is a
format string like any other, and renders as sent. A quoted heredoc, <<'EOF', passes $,
a backslash or a quote as written.
With --format json, ndjson, csv or sql the template must be a record — a JSON
template whose fields are its columns, or a lone reference such as {/path}:
one reference and nothing else, with one newline after it allowed — and the rows are
written as one JSON array, one JSON object per line, one CSV row (after a header), or
one INSERT.
-d, --data-path D a data directory to layer over the shipped data (repeatable; last wins on a clash)
--format F output form: text (default), json, ndjson, csv or sql
-h, --help print this help, then exit
--list list the paths the data offers, each written after {/ in a template, then exit;
--repeat, --separator, --format and --table do nothing here
--no-shipped-data load only the --data-path directories
-n, --repeat N render the template N times, 0..1048576 (default 1); 0 prints nothing,
[] under --format json, or the header under --format csv
-s, --seed N same seed, version and data: identical output (default: a random seed)
--separator S string between repeated renders (default empty); refused with a record
--format, which has its own separators
--table T the INSERT target for --format sql, ignored under any other
(default: a lone reference's last segment, else records)
--version print the version, then exit
A short flag's value attaches or follows (-n3, -n 3); short flags bundle (-hn 3);
-- ends the flags.
Exit codes: 0 success, 1 runtime error, 2 misuse.
`
type invocation struct {
args []string
dirs []string
format string
help bool
list bool
noShipped bool
repeat int
seed uint64
seeded bool
separator string
separatorSet bool
table string
tableSet bool
version bool
}
type flagDef struct {
long string
short string
value bool
set func(*invocation, string) error
}
var flagDefs = []flagDef{
{"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }},
{"format", "", true, func(in *invocation, v string) error { in.format = v; return nil }},
{"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }},
{"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }},
{"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }},
{"repeat", "n", true, func(in *invocation, v string) error {
n, err := strconv.Atoi(v)
if err != nil || n < 0 || n > fejkdata.MaxRepeat {
return fmt.Errorf("--repeat needs an integer in 0..%d, got %q", fejkdata.MaxRepeat, v)
}
in.repeat = n
return nil
}},
{"seed", "s", true, func(in *invocation, v string) error {
n, err := strconv.ParseUint(v, 10, 64)
if err != nil {
return fmt.Errorf("--seed needs an unsigned integer, got %q", v)
}
in.seed, in.seeded = n, true
return nil
}},
{"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }},
{"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }},
{"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }},
}
func flagByLong(name string) *flagDef {
for i := range flagDefs {
if flagDefs[i].long == name {
return &flagDefs[i]
}
}
return nil
}
func flagByShort(name string) *flagDef {
for i := range flagDefs {
if flagDefs[i].short == name {
return &flagDefs[i]
}
}
return nil
}
// flagArg is one flag as written: its definition and the value attached to it,
// if any.
type flagArg struct {
def *flagDef
value string
inline bool
}
// splitFlags reads one argv element as flags: --name or --name=value, or -abc
// where each letter is a short flag and the first that takes a value takes the
// rest of the element.
func splitFlags(arg string) ([]flagArg, error) {
if strings.HasPrefix(arg, "--") {
name, value, inline := strings.Cut(arg[2:], "=")
def := flagByLong(name)
if def == nil {
return nil, fmt.Errorf("unknown flag --%s", name)
}
return []flagArg{{def, value, inline}}, nil
}
if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil {
return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name)
}
var flags []flagArg
letters := arg[1:]
for i, r := range letters {
letter := string(r)
def := flagByShort(letter)
if def == nil {
return nil, fmt.Errorf("unknown flag -%s", letter)
}
if !def.value {
flags = append(flags, flagArg{def: def})
continue
}
rest := letters[i+len(letter):]
if strings.HasPrefix(rest, "=") {
return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s",
letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:])
}
return append(flags, flagArg{def, rest, rest != ""}), nil
}
return flags, nil
}
func parseArgs(argv []string) (invocation, error) {
in := invocation{repeat: 1, format: "text"}
for i := 0; i < len(argv); i++ {
arg := argv[i]
if arg == "--" {
in.args = append(in.args, argv[i+1:]...)
return in, nil
}
if len(arg) < 2 || arg[0] != '-' {
in.args = append(in.args, arg)
continue
}
flags, err := splitFlags(arg)
if err != nil {
return in, err
}
for _, fl := range flags {
if !fl.def.value {
if fl.inline {
return in, fmt.Errorf("--%s takes no value", fl.def.long)
}
_ = fl.def.set(&in, "")
continue
}
value := fl.value
if !fl.inline {
if i++; i >= len(argv) {
return in, fmt.Errorf("--%s needs a value", fl.def.long)
}
value = argv[i]
}
if err := fl.def.set(&in, value); err != nil {
return in, err
}
}
}
return in, nil
}
// recordFormat is one way to write a record out: the line each record renders,
// the header that precedes the first one, and the open/close frame plus the
// between-record separator a document form needs.
type recordFormat struct {
header func(*fejkdata.Record) string
line func(r *fejkdata.Record, table string) string
open string
close string
sep string
writes string // what it writes between values, which --separator would seem to set
}
// recordFormats is every --format that writes records. json frames the records
// as one array document; ndjson writes one object per line.
var recordFormats = map[string]recordFormat{
"csv": {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }, sep: "\n", writes: `always separates fields with ","`},
"json": {line: jsonLine, open: "[", close: "]", sep: ",\n", writes: "writes the records as one JSON array"},
"ndjson": {line: jsonLine, sep: "\n", writes: "writes one record per line"},
"sql": {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }, sep: "\n", writes: "writes one INSERT per line"},
}
func jsonLine(r *fejkdata.Record, _ string) string { return r.JSON() }
func (in invocation) writesRecords() bool {
return in.format != "text"
}
// formatNames lists the --format values, text included, for the misuse error.
func formatNames() string {
names := []string{"text"}
for name := range recordFormats {
names = append(names, name)
}
sort.Strings(names)
return strings.Join(names[:len(names)-1], ", ") + " or " + names[len(names)-1]
}
// checkFlags rejects a flag value or combination that cannot run.
func (in invocation) checkFlags() error {
if _, ok := recordFormats[in.format]; !ok && in.format != "text" {
return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format)
}
if in.tableSet && in.table == "" && in.format == "sql" {
return errors.New("--table names the INSERT target, so it cannot be empty")
}
if f, records := recordFormats[in.format]; records && in.separatorSet && !in.list {
return fmt.Errorf("--separator joins text renders; --format %s %s", in.format, f.writes)
}
return nil
}
func pipeHint(name string) string { return "echo 'id: {int(1,999)}' | " + name }
// operandError refuses an argument that is no flag, naming the spelling that reads what
// it most likely means: dropping it for -, a file for a file's name, else a template
// piped in.
func operandError(arg, name string) error {
unexpected := "unexpected argument '" + arg + "'"
if arg == "-" {
return fmt.Errorf("%s; stdin is read already, so drop it", unexpected)
}
if fi, err := os.Stat(arg); err == nil && fi.Mode().IsRegular() {
return fmt.Errorf("%s; read a file from stdin: %s < %s", unexpected, name, shellQuoted(arg))
}
ref := "{/" + strings.TrimPrefix(arg, "/") + "}"
_, argLone := loneReference(arg)
inline, err := fejkdata.IsTemplate(arg)
_, refLone := loneReference(ref)
example := pipeHint(name)
switch {
case argLone || inline:
example = piped(arg, name)
case err == nil && refLone:
example = piped(ref, name)
}
return fmt.Errorf("%s; the template is read from stdin: %s", unexpected, example)
}
// check rejects a flag combination or an operand that cannot run, before any data is
// read.
func (in invocation) check(name string) error {
if err := in.checkFlags(); err != nil {
return err
}
if len(in.args) > 0 {
return operandError(in.args[0], name)
}
return nil
}
func (in invocation) options(modules []fs.FS, systemSeed uint64) []fejkdata.Option {
if in.noShipped {
modules = nil
}
opts := []fejkdata.Option{fejkdata.WithDataFS(modules...)}
for _, dir := range in.dirs {
opts = append(opts, fejkdata.WithDataPath(dir))
}
seed := in.seed
if !in.seeded {
seed = systemSeed
}
return append(opts, fejkdata.WithSeed(seed))
}
// write streams the input's renders to w, repeat of them joined by the separator; a write
// failure surfaces from Flush, bufio keeping the first one.
func (in invocation) write(f *fejkdata.Generator, src input, w io.Writer) error {
if in.writesRecords() {
return in.writeRecords(f, src, w)
}
t, err := f.NewTemplate(src.template)
if err != nil {
return templateError(err)
}
out := bufio.NewWriter(w)
for i := 0; i < in.repeat; i++ {
if i > 0 {
out.WriteString(in.separator)
}
out.WriteString(t.Fake())
}
return out.Flush()
}
// recordStream builds the record drawer for the input, plus the INSERT table
// a sql format names.
func (in invocation) recordStream(f *fejkdata.Generator, src input) (func() (*fejkdata.Record, error), string, error) {
record := func() (*fejkdata.Record, error) { return f.FakeRecord(src.record) }
if src.record == "" {
t, err := f.NewRecordTemplate(src.template)
if err != nil {
return nil, "", templateError(spacedReference(src.template, err))
}
record = func() (*fejkdata.Record, error) { return t.Fake(), nil }
}
table := in.table
if table == "" {
table = defaultTable(src.record)
}
return record, table, nil
}
// writeRecords streams a record per line in the chosen format, framing a document
// form with its open/close brackets and a header preceding the first record.
func (in invocation) writeRecords(f *fejkdata.Generator, src input, w io.Writer) error {
record, table, err := in.recordStream(f, src)
if err != nil {
return err
}
format := recordFormats[in.format]
if in.repeat == 0 {
return writeNoRecord(record, format, w)
}
out := bufio.NewWriter(w)
if format.open != "" {
out.WriteString(format.open)
out.WriteByte('\n')
}
for i := 0; i < in.repeat; i++ {
r, err := record()
if err != nil {
return err
}
if i == 0 && format.header != nil {
out.WriteString(format.header(r))
out.WriteByte('\n')
}
if i > 0 {
out.WriteString(format.sep)
}
out.WriteString(format.line(r, table))
}
if format.close != "" {
out.WriteByte('\n')
out.WriteString(format.close)
}
out.WriteByte('\n')
return out.Flush()
}
// writeNoRecord writes zero records: the header where the format writes one, an empty document
// where it frames one, else nothing. It draws one record, which names the header's columns, so a
// lone reference to nothing fails as it would with records.
func writeNoRecord(record func() (*fejkdata.Record, error), format recordFormat, w io.Writer) error {
r, err := record()
switch {
case err != nil:
return err
case format.header != nil:
_, err = io.WriteString(w, format.header(r)+"\n")
case format.open != "":
_, err = io.WriteString(w, format.open+format.close+"\n")
}
return err
}
// defaultTable names the INSERT target when --table is absent: the path's last
// segment, or "records" when stdin is not a lone reference.
func defaultTable(path string) string {
if path == "" {
return "records"
}
segments := strings.Split(withoutSelectors(path), ".")
return segments[len(segments)-1]
}
// withoutSelectors is path with its [selectors] cut out.
func withoutSelectors(path string) string {
var names strings.Builder
for depth, i := 0, 0; i < len(path); i++ {
switch {
case path[i] == '[':
depth++
case path[i] == ']':
depth--
case depth == 0:
names.WriteByte(path[i])
}
}
return names.String()
}
// inputError marks a failure that is the input's own fault — nothing piped in, a template
// that does not compile, or a stdin that cannot be read. run reports it as misuse (exit 2, with a
// pointer to --help), unlike a lone reference to nothing under --format, which is a runtime
// error (exit 1).
type inputError struct{ error }
func (e inputError) Unwrap() error { return e.error }
// jsonSpace is the whitespace JSON allows around a value.
const jsonSpace = " \t\r\n"
// input is what stdin holds: the template, and the path a lone reference names as a
// record.
type input struct {
template string
record string
}
// parseInput keeps stdin whole as the template, and names a record where stdin is a lone
// reference once JSON's surrounding whitespace, or one final newline, is cut.
func parseInput(raw string) input {
trimmed := strings.Trim(raw, jsonSpace)
text := raw
if json.Valid([]byte(trimmed)) {
text = trimmed
}
text, found := strings.CutSuffix(text, "\n")
if found {
text = strings.TrimSuffix(text, "\r")
}
record, _ := loneReference(text)
return input{template: raw, record: record}
}
// loneReference returns the path that text reads when it is one reference from the root
// and nothing else, bare or as a JSON string.
func loneReference(text string) (string, bool) {
var unquoted string
if strings.HasPrefix(text, `"`) && strings.HasSuffix(text, `"`) && json.Unmarshal([]byte(text), &unquoted) == nil {
text = unquoted
}
path, opens := strings.CutPrefix(text, "{/")
path, closes := strings.CutSuffix(path, "}")
if !opens || !closes || !isPath(path) {
return "", false
}
return path, true
}
// isPath reports whether a reference's body after its / is a path alone: no second /, no
// folder sigil, no empty segment, and none of the |, ( or " as " a token reads outside a selector.
// It copies the grammar, since the CLI reads only the library's public API.
func isPath(body string) bool {
if body == "" || strings.ContainsAny(body[:1], "/.") {
return false
}
names := withoutSelectors(body)
if strings.ContainsAny(names, "|(") || strings.Contains(names, " as ") || hasEmptySegment(names) {
return false
}
inline, err := fejkdata.IsTemplate(body)
return !inline && err == nil
}
// hasEmptySegment reports whether a dotted path holds a segment with no name: two dots are a
// step up, and only a step up may end the path.
func hasEmptySegment(path string) bool {
segments := strings.Split(path, ".")
stepUp := false
for i, seg := range segments[1:] {
last := i == len(segments)-2
switch {
case seg != "":
stepUp = false
case stepUp != last:
return true
default:
stepUp = !stepUp
}
}
return false
}
// spacedReference adds to a record refusal that whitespace around a lone reference made
// the template text.
func spacedReference(template string, err error) error {
if trimmed := strings.TrimSpace(template); trimmed != template && errors.Is(err, fejkdata.ErrNoColumns) {
if path, lone := loneReference(trimmed); lone {
return fmt.Errorf("%w; the whitespace around {/%s} makes it text, so remove it to write the record %s", err, path, path)
}
}
return err
}
// piped is the command piping template into the command called name; some shells' echo
// reads a backslash as an escape, so a template holding one goes through printf.
func piped(template, name string) string {
if strings.ContainsRune(template, '\\') {
return "printf '%s' " + shellQuoted(template) + " | " + name
}
return "echo " + shellQuoted(template) + " | " + name
}
// shellQuoted is s as a shell reads it back: bare where it holds nothing a shell expands.
func shellQuoted(s string) string {
if s != "" && !strings.ContainsAny(s, " \t\r\n'\"$`\\*?[]{}()<>|&;#~!") {
return s
}
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
}
// Main runs the command on the process's arguments and standard streams over modules, each
// passed to [fejkdata.WithDataFS], and returns the exit code: 0 ok, 1 runtime error,
// 2 misuse. --no-shipped-data leaves modules out.
func Main(modules ...fs.FS) int {
var stdin io.Reader = os.Stdin
if fi, err := os.Stdin.Stat(); err == nil && fi.Mode()&os.ModeCharDevice != 0 {
stdin = nil
}
return run(commandName(os.Args[0]), modules, os.Args[1:], rand.Uint64(), stdin, os.Stdout, os.Stderr)
}
// commandName is the name the command runs under: the file arg0 names, less a Windows
// .exe, or fejkdata where arg0 names none.
func commandName(arg0 string) string {
name := filepath.Base(arg0)
if ext := filepath.Ext(name); strings.EqualFold(ext, ".exe") {
name = strings.TrimSuffix(name, ext)
}
if name == "." || name == "" {
return "fejkdata"
}
return name
}
// run is Main over the given streams, run as the command name, seeded by systemSeed where
// no --seed is given. A nil stdin is a terminal, which holds no template.
func run(name string, modules []fs.FS, args []string, systemSeed uint64, stdin io.Reader, stdout, stderr io.Writer) int {
in, err := parseArgs(args)
if err != nil {
return misuse(stderr, name, err)
}
if in.help {
io.WriteString(stdout, strings.ReplaceAll(usage, "COMMAND", name))
return 0
}
if in.version {
info, _ := debug.ReadBuildInfo()
fmt.Fprintln(stdout, versionLine(name, info))
return 0
}
if err := in.check(name); err != nil {
return misuse(stderr, name, err)
}
var src input
if !in.list {
if src, err = readStdin(stdin, name); err != nil {
return fail(stderr, name, err)
}
}
f, err := fejkdata.New(in.options(modules, systemSeed)...)
if err != nil {
return fail(stderr, name, err)
}
if in.list {
for _, p := range f.List() {
fmt.Fprintln(stdout, p)
}
return 0
}
if err := in.write(f, src, stdout); err != nil {
return fail(stderr, name, err)
}
return 0
}
func readStdin(stdin io.Reader, name string) (input, error) {
if stdin == nil {
return input{}, inputError{fmt.Errorf("the template comes from stdin, and nothing is piped in: %s; %s --list prints the paths a {/path} reference reads", pipeHint(name), name)}
}
raw, err := io.ReadAll(stdin)
if err != nil {
return input{}, inputError{fmt.Errorf("stdin cannot be read: %w", err)}
}
return parseInput(string(raw)), nil
}
// templateError is misuse unless the data the template reads failed to load.
func templateError(err error) error {
if errors.Is(err, fejkdata.ErrLoad) {
return err
}
return inputError{err}
}
func fail(stderr io.Writer, name string, err error) int {
var te inputError
if errors.As(err, &te) {
return misuse(stderr, name, te.error)
}
report(stderr, name, err)
if errors.Is(err, fejkdata.ErrNoColumns) {
fmt.Fprintln(stderr, "wrap that JSON in single quotes, which keep a shell from expanding its braces")
}
return 1
}
func misuse(stderr io.Writer, name string, err error) int {
report(stderr, name, err)
fmt.Fprintf(stderr, "try '%s --help'\n", name)
return 2
}
// report writes err under the command name, in place of the library's own on each line.
func report(stderr io.Writer, name string, err error) {
msg := strings.ReplaceAll(strings.TrimPrefix(err.Error(), "fejkdata: "), "\nfejkdata: ", "\n"+name+": ")
fmt.Fprintf(stderr, "%s: %s\n", name, msg)
}
// fejkdataModule is the path of the module holding the library; the root package's path is
// the module's.
var fejkdataModule = reflect.TypeOf((*fejkdata.Generator)(nil)).Elem().PkgPath()
// versionLine names the fejkdata version the binary is built with. Where another module's
// main package carries the CLI, the main module's version comes first, under the command name.
func versionLine(name string, info *debug.BuildInfo) string {
if info == nil {
return "fejkdata devel"
}
if info.Main.Path == fejkdataModule {
return "fejkdata " + moduleVersion(info.Main)
}
for _, m := range info.Deps {
if m.Path == fejkdataModule {
return name + " " + moduleVersion(info.Main) + ", fejkdata " + moduleVersion(*m)
}
}
return name + " " + moduleVersion(info.Main) + ", fejkdata devel"
}
// moduleVersion is the version m is built at: the replacement's where a replace directive
// names one, and "devel" where it carries none, as a local directory does.
func moduleVersion(m debug.Module) string {
if m.Replace != nil {
m = *m.Replace
}
if m.Version == "" || m.Version == "(devel)" {
return "devel"
}
return m.Version
}
+961
View File
@@ -0,0 +1,961 @@
package cli
import (
"bytes"
"encoding/csv"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"regexp"
"runtime/debug"
"strings"
"testing"
"testing/fstest"
)
// shipped holds the shipped modules as read from the checkout, each folder under ../data
// holding a manifest, so these tests link no data package.
var shipped = func() []fs.FS {
var modules []fs.FS
err := filepath.WalkDir(filepath.Join("..", "data"), func(path string, d fs.DirEntry, err error) error {
if err == nil && d.Name() == ".fejkdata.json" {
modules = append(modules, os.DirFS(filepath.Dir(path)))
}
return err
})
if err != nil || len(modules) == 0 {
panic(fmt.Sprintf("no shipped module found under ../data: %v", err))
}
return modules
}()
// svSE, enUS and misc are copies of a shipped module's files without its manifest, so a test
// layers its categories unprefixed with -d.
var svSE, enUS, misc string
func TestMain(m *testing.M) {
tmp, err := os.MkdirTemp("", "fejkdata-cli-test")
if err != nil {
panic(err)
}
svSE, enUS, misc = bareCopy(tmp, "sv_SE"), bareCopy(tmp, "en_US"), bareCopy(tmp, "misc")
code := m.Run()
os.RemoveAll(tmp)
os.Exit(code)
}
// bareCopy copies the category and rows files of the shipped module in data/<module> into a
// folder under tmp, and returns that folder.
func bareCopy(tmp, module string) string {
src, dst := filepath.Join("..", "data", module), filepath.Join(tmp, module)
if err := os.Mkdir(dst, 0o755); err != nil {
panic(err)
}
for _, pattern := range []string{"*.json", "*.tsv"} {
files, err := filepath.Glob(filepath.Join(src, pattern))
if err != nil {
panic(err)
}
for _, f := range files {
if strings.HasPrefix(filepath.Base(f), ".") {
continue
}
b, err := os.ReadFile(f)
if err == nil {
err = os.WriteFile(filepath.Join(dst, filepath.Base(f)), b, 0o644)
}
if err != nil {
panic(err)
}
}
}
return dst
}
func runOut(stdin string, args ...string) (int, string, string) {
var out, errb bytes.Buffer
code := run("fejkdata", shipped, args, 1, strings.NewReader(stdin), &out, &errb)
return code, out.String(), errb.String()
}
// runTerminal runs as from a shell with nothing piped in.
func runTerminal(args ...string) (int, string, string) {
var out, errb bytes.Buffer
code := run("fejkdata", shipped, args, 1, nil, &out, &errb)
return code, out.String(), errb.String()
}
func TestRunLoadsTheModulesItIsGiven(t *testing.T) {
module := fstest.MapFS{"pet.json": {Data: []byte(`{"format":"{name}","name":["Fido"]}`)}}
var out, errb bytes.Buffer
if code := run("fejkdata", []fs.FS{module}, nil, 1, strings.NewReader("{/pet}"), &out, &errb); code != 0 || out.String() != "Fido" {
t.Errorf("run over a module holding pet = %d, %q, %q, want Fido", code, out.String(), errb.String())
}
out.Reset()
if code := run("fejkdata", []fs.FS{module}, []string{"--list"}, 1, nil, &out, &errb); code != 0 || out.String() != "pet\npet.name\n" {
t.Errorf("--list over a module holding pet = %d, %q, want pet and pet.name only", code, out.String())
}
}
func TestRunOutputsValue(t *testing.T) {
code, out, errb := runOut("{/person}", "--data-path", svSE)
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
if out == "" || strings.Contains(out, "\n") {
t.Fatalf("output = %q, stderr=%q; want one value and nothing added", out, errb)
}
}
func TestRunDotPath(t *testing.T) {
code, full, _ := runOut("{/person}", "--seed", "7", "--data-path", svSE)
if code != 0 {
t.Fatalf("person run = %d", code)
}
code, last, errb := runOut("{/person.last}", "--seed", "7", "--data-path", svSE)
if code != 0 {
t.Fatalf("person.last run = %d, stderr=%q", code, errb)
}
if strings.TrimSpace(last) == "" {
t.Fatal("empty person.last output")
}
if last == full {
t.Errorf("person.last %q should differ from person %q", last, full)
}
}
func TestRunWithoutASeedTakesTheSystemSeed(t *testing.T) {
var drawn, seeded, errb bytes.Buffer
if code := run("fejkdata", shipped, nil, 42, strings.NewReader("{uuid()}"), &drawn, &errb); code != 0 {
t.Fatalf("run with system seed 42 = %d, stderr=%q", code, errb.String())
}
if code := run("fejkdata", shipped, []string{"--seed", "42"}, 7, strings.NewReader("{uuid()}"), &seeded, &errb); code != 0 {
t.Fatalf("run --seed 42 = %d, stderr=%q", code, errb.String())
}
if drawn.String() != seeded.String() {
t.Fatalf("run with system seed 42 = %q, run --seed 42 = %q; want the same draw", drawn.String(), seeded.String())
}
}
func TestRunSeedSpellings(t *testing.T) {
_, want, _ := runOut("{/address}", "--seed", "42", "--data-path", svSE)
for _, args := range [][]string{
{"--seed=42", "--data-path", svSE},
{"-s", "42", "-d", svSE},
{"--data-path", svSE, "--seed", "42"},
} {
code, got, errb := runOut("{/address}", args...)
if code != 0 {
t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
}
if got != want {
t.Errorf("run(%v) = %q, want %q", args, got, want)
}
}
}
func TestRunShortFlagValues(t *testing.T) {
_, want, _ := runOut("{/address}", "--seed", "42", "--data-path", svSE)
for _, args := range [][]string{
{"-s42", "-d", svSE},
{"-s", "42", "-d" + svSE},
{"-d", svSE, "-s42"},
} {
code, got, errb := runOut("{/address}", args...)
if code != 0 {
t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
}
if got != want {
t.Errorf("run(%v) = %q, want %q", args, got, want)
}
}
_, three, _ := runOut("{/word}\n", "-s", "1", "-n3", "-d", svSE)
if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 {
t.Errorf("-n3 gave %d lines: %q", len(lines), three)
}
for _, c := range []struct {
args []string
want string
}{
{[]string{"-s=42", "-d", svSE}, "-s42"},
{[]string{"-s=42", "-d", svSE}, "--seed=42"},
{[]string{"-d=" + svSE}, "--data-path="},
{[]string{"-nd", "3", "-d", svSE}, `--repeat needs an integer in 0..1048576, got "d"`},
{[]string{"--seed=", "-d", svSE}, `--seed needs an unsigned integer, got ""`},
} {
code, out, errb := runOut("{/word}", c.args...)
if code != 2 || out != "" {
t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out)
}
if !strings.Contains(errb, c.want) {
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
}
}
code, out, _ := runOut("", "-hd", svSE)
if code != 0 || !strings.Contains(out, "Usage") {
t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out)
}
}
func TestRunDoubleDashEndsFlags(t *testing.T) {
code, out, errb := runOut("{/person}", "--data-path", svSE, "--")
if code != 0 || strings.TrimSpace(out) == "" {
t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
}
code, _, errb = runOut("{/person}", "--data-path", svSE, "--", "--list")
if code != 2 || !strings.Contains(errb, "stdin") {
t.Errorf("after --, --list should be an argument: code %d, stderr %q", code, errb)
}
}
func TestRunSingleDashLongIsRejected(t *testing.T) {
for _, c := range []struct {
args []string
want string
}{
{[]string{"-seed", "42", "-d", svSE}, "use --seed"},
{[]string{"-seed=42", "-d", svSE}, "use --seed"},
{[]string{"-data-path", svSE}, "use --data-path"},
{[]string{"-list", "-d", svSE}, "use --list"},
{[]string{"--nope", "-d", svSE}, "unknown flag --nope"},
{[]string{"-x", "-d", svSE}, "unknown flag -x"},
} {
code, _, errb := runOut("{/person}", c.args...)
if code != 2 {
t.Errorf("run(%v) = %d, want 2", c.args, code)
}
if !strings.Contains(errb, c.want) {
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
}
}
}
func TestRunFlagValues(t *testing.T) {
for _, c := range []struct {
args []string
want string
}{
{[]string{"--seed", "-d", svSE}, `--seed needs an unsigned integer, got "-d"`},
{[]string{"-d", svSE, "--seed"}, "--seed needs a value"},
{[]string{"--seed", "x", "-d", svSE}, "--seed"},
{[]string{"--list=1", "-d", svSE}, "--list takes no value"},
{[]string{"--repeat", "-1", "-d", svSE}, "--repeat"},
{[]string{"-n", "-1", "-d", svSE}, "--repeat"},
} {
code, _, errb := runOut("{/person}", c.args...)
if code != 2 {
t.Errorf("run(%v) = %d, want 2", c.args, code)
}
if !strings.Contains(errb, c.want) {
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
}
}
}
func TestRunRepeat(t *testing.T) {
code, out, errb := runOut("{/word}\n", "--seed", "1", "--repeat", "3", "--data-path", svSE)
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
if len(lines) != 3 {
t.Errorf("repeat=3 gave %d lines: %q", len(lines), out)
}
_, short, _ := runOut("{/word}\n", "--seed", "1", "-n", "3", "-d", svSE)
if short != out {
t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out)
}
}
func TestRunSeparator(t *testing.T) {
code, out, errb := runOut("{/word}", "--repeat", "3", "--separator", ",", "--data-path", svSE)
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
if n := strings.Count(out, "\n"); n != 0 {
t.Errorf("want no newline, got %d: %q", n, out)
}
if !strings.Contains(out, ",") {
t.Errorf("values should be comma-joined: %q", out)
}
}
func TestRunRepeatAdvancesRNG(t *testing.T) {
code, out, errb := runOut("{/person}\n", "--seed", "1", "--repeat", "5", "--data-path", svSE)
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
uniq := map[string]bool{}
for _, l := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
uniq[l] = true
}
if len(uniq) < 2 {
t.Errorf("repeat should vary output, all identical: %q", out)
}
}
func TestRunMisuse(t *testing.T) {
for _, c := range []struct {
stdin string
args []string
}{
{"{/person}", []string{"-d", svSE, "word"}},
{"", []string{"-d", svSE, "--list", "person"}},
{"{/sv_SE.person}", []string{"--no-shipped-data"}},
} {
code, out, errb := runOut(c.stdin, c.args...)
if code != 2 {
t.Errorf("run(%q, %v) = %d, want 2", c.stdin, c.args, code)
}
if !strings.Contains(errb, "try 'fejkdata --help'") {
t.Errorf("run(%q, %v) stderr = %q, want a pointer to --help", c.stdin, c.args, errb)
}
if out != "" {
t.Errorf("run(%q, %v) stdout = %q, want nothing", c.stdin, c.args, out)
}
}
}
func TestRunTakesNoOperand(t *testing.T) {
file := filepath.Join(t.TempDir(), "users.tmpl")
if err := os.WriteFile(file, []byte("{/sv_SE.person}"), 0o644); err != nil {
t.Fatal(err)
}
for _, c := range []struct {
args []string
want string
}{
{[]string{"sv_SE.person"}, "echo '{/sv_SE.person}' | fejkdata"},
{[]string{"/en_US.address"}, "echo '{/en_US.address}' | fejkdata"},
{[]string{"{/en_US.person}"}, "echo '{/en_US.person}' | fejkdata"},
{[]string{"name: {/sv_SE.person.last}"}, "echo 'name: {/sv_SE.person.last}' | fejkdata"},
{[]string{"geo.US.locality[O'Fallon].name"}, `echo '{/geo.US.locality[O'\''Fallon].name}' | fejkdata`},
{[]string{"en_US.address"}, "echo '{/en_US.address}' | fejkdata"},
{[]string{`C:\temp {/sv_SE.person}`}, `printf '%s' 'C:\temp {/sv_SE.person}' | fejkdata`},
{[]string{"-"}, "drop it"},
{[]string{file}, "fejkdata < " + file},
} {
code, out, errb := runOut("", c.args...)
if code != 2 || out != "" || !strings.Contains(errb, "unexpected argument") || !strings.Contains(errb, c.want) {
t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
}
}
}
type failingReader struct{}
func (failingReader) Read([]byte) (int, error) { return 0, errors.New("bad file descriptor") }
func TestRunUnreadableStdinIsMisuse(t *testing.T) {
var out, errb bytes.Buffer
if code := run("fejkdata", shipped, nil, 1, failingReader{}, &out, &errb); code != 2 || !strings.Contains(errb.String(), "cannot be read") {
t.Errorf("run with an unreadable stdin = %d, %q; want misuse naming it", code, errb.String())
}
}
func TestRunTerminalStdinIsMisuse(t *testing.T) {
code, out, errb := runTerminal("--seed", "1")
if code != 2 || out != "" || !strings.Contains(errb, "nothing is piped in: echo 'id: {int(1,999)}' | fejkdata; fejkdata --list prints the paths a {/path} reference reads\n") {
t.Errorf("run with nothing piped = %d, %q, %q; want misuse showing a template piped to fejkdata", code, out, errb)
}
for _, args := range [][]string{{"--list"}, {"--help"}, {"--version"}} {
if code, out, errb := runTerminal(args...); code != 0 || out == "" {
t.Errorf("run(%v) with nothing piped = %d, %q, %q; want it to run, reading no stdin", args, code, out, errb)
}
}
}
func TestRunRendersExactlyWhatIsSent(t *testing.T) {
for stdin, want := range map[string]string{
"hihi": "hihi",
"hihi\n": "hihi\n",
"hihi\r\n": "hihi\r\n",
"hihi\n\n": "hihi\n\n",
"{{x}}\n": "{x}\n",
"\"a\"\n": "a",
" \t\"a\"\r\n\n": "a",
"{/misc.territory[SE].alpha2}": "SE",
"{/misc.territory[SE].alpha2}\n": "SE\n",
} {
code, out, errb := runOut(stdin, "--seed", "1")
if code != 0 || out != want {
t.Errorf("run(%q) = %d, %q, %q; want %q", stdin, code, out, errb, want)
}
}
if code, out, errb := runOut("{\"format\":\"{x}\\n\",\"x\":[\"a\",\"b\"]}\n"); code != 0 || (out != "a\n" && out != "b\n") {
t.Errorf("a JSON template's format = %d, %q, %q; want its own newline only", code, out, errb)
}
if code, out, errb := runOut("{/sv_SE.word}\n", "--seed", "1", "-n", "3", "--separator", ","); code != 0 || strings.Count(out, "\n,") != 2 || !strings.HasSuffix(out, "\n") {
t.Errorf("echo's newline and a separator = %d, %q, %q; want each render's newline, then the separator", code, out, errb)
}
}
func TestRunRendersBlankStdin(t *testing.T) {
for _, stdin := range []string{"", "\n", " \r\n\t"} {
if code, out, errb := runOut(stdin); code != 0 || out != stdin {
t.Errorf("run(%q) = %d, %q, %q; want the blank template rendered as it is", stdin, code, out, errb)
}
}
}
func TestRunTextPrintsAsWritten(t *testing.T) {
for _, stdin := range []string{"sv_SE.person", "[Skåne län]", "x[1]y", `say "hi"`} {
code, out, errb := runOut(stdin)
if code != 0 || out != stdin {
t.Errorf("run(%q) = %d, %q, %q; want the text as written", stdin, code, out, errb)
}
}
}
func TestRunMultipleDataPaths(t *testing.T) {
code, out, errb := runOut("{/person}", "-d", enUS, "--data-path", svSE)
if code != 0 {
t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb)
}
if strings.TrimSpace(out) == "" {
t.Fatal("empty output for multi-dir run")
}
}
func TestRunUnknownCategoryFails(t *testing.T) {
code, _, errb := runOut("{/nope}", "--data-path", svSE, "--format", "csv")
if code != 1 {
t.Fatalf("run = %d, want 1", code)
}
if !strings.Contains(errb, "nope") {
t.Errorf("stderr %q should name the unknown category", errb)
}
}
func TestRunList(t *testing.T) {
code, out, errb := runOut("", "--data-path", svSE, "--list")
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
for _, want := range []string{"person", "person.last", "address"} {
if !strings.Contains(out, want) {
t.Errorf("list output missing %q:\n%s", want, out)
}
}
}
func TestRunHelpOnStdout(t *testing.T) {
for _, h := range []string{"-h", "--help"} {
code, out, errb := runOut("", h)
if code != 0 {
t.Errorf("%s = %d, want 0", h, code)
}
if !strings.Contains(out, "Usage") || errb != "" {
t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb)
}
}
}
func TestVersionLineNamesFejkdatasVersion(t *testing.T) {
fejkdata := func(version string, replace *debug.Module) *debug.Module {
return &debug.Module{Path: "github.com/larvit/fejkdata", Version: version, Replace: replace}
}
mycli := debug.Module{Path: "example.com/mycli", Version: "v9.0.0"}
for _, c := range []struct {
info *debug.BuildInfo
want string
}{
{&debug.BuildInfo{Main: *fejkdata("v0.1.0", nil)}, "fejkdata v0.1.0"},
{&debug.BuildInfo{Main: *fejkdata("(devel)", nil)}, "fejkdata devel"},
{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", nil)}}, "mycli v9.0.0, fejkdata v0.1.0"},
{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", &debug.Module{Path: "../fejkdata"})}}, "mycli v9.0.0, fejkdata devel"},
{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", &debug.Module{Path: "example.com/fork", Version: "v0.1.1"})}}, "mycli v9.0.0, fejkdata v0.1.1"},
{&debug.BuildInfo{Main: mycli}, "mycli v9.0.0, fejkdata devel"},
{nil, "fejkdata devel"},
} {
if got := versionLine("mycli", c.info); got != c.want {
t.Errorf("versionLine(mycli, %+v) = %q, want %q", c.info, got, c.want)
}
}
}
func TestCommandNameIsTheFileItRunsFrom(t *testing.T) {
for arg0, want := range map[string]string{
"/usr/local/bin/mycli": "mycli",
"mycli.EXE": "mycli",
"mycli.exe": "mycli",
"my.cli": "my.cli",
"": "fejkdata",
} {
if got := commandName(arg0); got != want {
t.Errorf("commandName(%q) = %q, want %q", arg0, got, want)
}
}
}
func TestRunNamesItselfByTheCommandName(t *testing.T) {
var out, errb bytes.Buffer
if code := run("mycli", shipped, []string{"--nope"}, 1, nil, &out, &errb); code != 2 || errb.String() != "mycli: unknown flag --nope\ntry 'mycli --help'\n" {
t.Errorf("mycli --nope = %d, %q", code, errb.String())
}
errb.Reset()
if code := run("mycli", shipped, nil, 1, nil, &out, &errb); code != 2 || !strings.Contains(errb.String(), "| mycli; mycli --list prints") {
t.Errorf("mycli with nothing piped = %d, %q, want a pipe into mycli", code, errb.String())
}
errb.Reset()
if code := run("mycli", []fs.FS{nil, nil}, nil, 1, strings.NewReader("x"), &out, &errb); code != 1 || strings.Count(errb.String(), "mycli: ") != 2 || strings.Contains(errb.String(), "fejkdata") {
t.Errorf("mycli over two nil modules = %d, %q, want each line under mycli", code, errb.String())
}
errb.Reset()
if code := run("mycli", shipped, []string{"-d", "/nope"}, 1, strings.NewReader("{/sv_SE.person}"), &out, &errb); code != 1 || !strings.HasPrefix(errb.String(), "mycli: ") || strings.Contains(errb.String(), "fejkdata") {
t.Errorf("mycli -d /nope = %d, %q, want a load error under mycli", code, errb.String())
}
if code := run("mycli", shipped, []string{"--help"}, 1, nil, &out, &errb); code != 0 || !strings.HasPrefix(out.String(), "Usage: <template> | mycli [flags]\n") || strings.Contains(out.String(), "fejkdata") {
t.Errorf("mycli --help = %d, %q, want mycli named throughout", code, out.String())
}
}
func TestRunVersion(t *testing.T) {
code, out, errb := runOut("", "--version")
if code != 0 {
t.Fatalf("--version = %d, stderr=%q", code, errb)
}
version, ok := strings.CutPrefix(out, "fejkdata ")
if !ok || strings.TrimSpace(version) == "" {
t.Errorf("--version = %q, want the command name and a version on stdout", out)
}
}
func TestRunMissingDirFails(t *testing.T) {
code, _, errb := runOut("{/person}", "--data-path", "../data/nope")
if code != 1 {
t.Fatalf("run = %d, want 1", code)
}
if errb == "" {
t.Error("want an error message on stderr")
}
}
func TestRunShippedDataByDefault(t *testing.T) {
code, out, errb := runOut("{/sv_SE.person}", "--seed", "1")
if code != 0 || strings.TrimSpace(out) == "" {
t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
}
code, list, _ := runOut("", "--list")
if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") {
t.Errorf("--list without --data-path = %d, %q", code, list)
}
code, _, errb = runOut("{/person}")
if code != 2 || !strings.Contains(errb, "person") {
t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb)
}
}
func TestRunSelectsATableRow(t *testing.T) {
dir := t.TempDir()
for name, content := range map[string]string{
"region.json": `{"format":"{name}","rows":"region.tsv","key":"code","name":"name"}`,
"region.tsv": "code\tname\n01\tStockholms län\n12\tSkåne län\n",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
if code, out, errb := runOut("{/region[12]}", "--no-shipped-data", "-d", dir); code != 0 || out != "Skåne län" {
t.Fatalf("{/region[12]} = %d, %q, stderr=%q", code, out, errb)
}
if code, out, _ := runOut("{/region[Skåne län].code}", "--no-shipped-data", "-d", dir); code != 0 || out != "12" {
t.Fatalf("{/region[Skåne län].code} = %d, %q", code, out)
}
if code, out, _ := runOut("{/region[12]}", "--no-shipped-data", "-d", dir, "--format", "sql"); code != 0 || out != `INSERT INTO "region" ("code", "name") VALUES ('12', 'Skåne län');`+"\n" {
t.Fatalf("--format sql {/region[12]} = %d, %q, want the table named without its selector", code, out)
}
if code, _, errb := runOut("{/region[99]}", "--no-shipped-data", "-d", dir); code != 2 || !strings.Contains(errb, `"99"`) {
t.Fatalf("{/region[99]} = %d, stderr=%q, want misuse naming the row", code, errb)
}
if code, out, _ := runOut("{/misc.territory[SE]}", "--seed", "1", "--format", "csv"); code != 0 || !strings.HasPrefix(out, "alpha2,") || !strings.Contains(out, "\nSE,SWE,") {
t.Fatalf("--format csv {/misc.territory[SE]} = %d, %q", code, out)
}
}
func TestUsageExamplesRenderWithNoData(t *testing.T) {
examples := regexp.MustCompile(`(?m)^ echo '([^']*)' \| COMMAND$`).FindAllStringSubmatch(usage+"\n "+pipeHint("COMMAND")+"\n", -1)
if len(examples) < 3 {
t.Fatalf("usage and pipe hint hold %d echo examples, want the help's two and the hint's", len(examples))
}
for _, e := range examples {
if code, out, errb := runOut(e[1], "--no-shipped-data"); code != 0 || strings.TrimSpace(out) == "" {
t.Errorf("echo %q with no data: run = %d, %q, stderr %q", e[1], code, out, errb)
}
}
for _, token := range regexp.MustCompile(`\{/[^}]*\}`).FindAllString(usage, -1) {
if token != "{/path}" && token != "{/table[KEY]}" {
t.Errorf("usage names %s, a path a custom CLI may not carry", token)
}
}
if !strings.Contains(usage, "\nExit codes: 0 success, 1 runtime error, 2 misuse.\n") {
t.Errorf("usage does not list the exit codes")
}
}
func TestRunOperandMisuseBeforeLoad(t *testing.T) {
code, _, errb := runOut("", "--no-shipped-data", "x")
if code != 2 || !strings.Contains(errb, "unexpected argument") || strings.Contains(errb, "--data-path") {
t.Fatalf("an operand with no data = %d, %q; want the operand refused before any load", code, errb)
}
}
func TestRunInlineTemplate(t *testing.T) {
code, out, errb := runOut("name: {/sv_SE.person.last}", "--seed", "1")
if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
t.Fatalf("inline format string = %d, %q, stderr %q", code, out, errb)
}
code, out, errb = runOut(`{"format":"name: {x}","x":["bosse","lina"]}`, "--seed", "1")
if code != 0 || (out != "name: bosse" && out != "name: lina") {
t.Fatalf("inline JSON template = %d, %q, want one name, stderr %q", code, out, errb)
}
code, out, errb = runOut(`"name: {/sv_SE.person.last}"`, "--seed", "1")
if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
t.Fatalf("inline JSON string = %d, %q, stderr %q", code, out, errb)
}
code, out, errb = runOut("{digits(1)}\n", "--seed", "1", "-n", "2")
if code != 0 || len(strings.Split(strings.TrimRight(out, "\n"), "\n")) != 2 {
t.Fatalf("inline template with --repeat = %d, %q, stderr %q", code, out, errb)
}
code, out, errb = runOut("{/sv_SE.person} hihi", "--seed", "1")
if code != 0 || !strings.HasSuffix(out, " hihi") || strings.Contains(out, "{") {
t.Fatalf("a reference and text = %d, %q, stderr %q", code, out, errb)
}
}
func TestRunFormatTakesOnlyALoneReferenceAsARecord(t *testing.T) {
for _, stdin := range []string{"{//sv_SE.person}", "{/sv_SE.person|/misc.uuid}", "{/sv_SE.person as p}", "{/}", "{/.sv_SE.person}"} {
if code, out, errb := runOut(stdin, "--format", "json"); code != 2 || out != "" {
t.Errorf("run(%q, --format json) = %d, %q, %q; want misuse", stdin, code, out, errb)
}
}
if code, out, errb := runOut(`{"format":"x"}`); code != 0 || out != "x" {
t.Errorf(`run({"format":"x"}) = %d, %q, %q; want x`, code, out, errb)
}
}
func TestRunTemplateMisuse(t *testing.T) {
for stdin, want := range map[string]string{
"{bad": "unterminated",
"name: {/no.such.path}": "no entry",
"a } b": "}}",
"{//sv_SE.person}": "write {/sv_SE.person}",
} {
code, out, errb := runOut(stdin, "--seed", "1")
if code != 2 || out != "" || !strings.Contains(errb, "try 'fejkdata --help'") || !strings.Contains(errb, want) {
t.Errorf("run(%q) = %d, %q, %q; want misuse naming %q and --help", stdin, code, out, errb, want)
}
if strings.Contains(errb, "fejkdata: fejkdata:") {
t.Errorf("run(%q) doubled the program prefix: %q", stdin, errb)
}
}
}
func TestRunNoShippedData(t *testing.T) {
code, list, errb := runOut("", "--no-shipped-data", "-d", misc, "--list")
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
if strings.Contains(list, "sv_SE") || !strings.Contains(list, "uuid\n") {
t.Errorf("--no-shipped-data --list = %q, want only the given dir", list)
}
code, out, _ := runOut("{/uuid}", "--no-shipped-data", "-d", misc, "-s", "3")
if code != 0 || strings.TrimSpace(out) == "" {
t.Errorf("run = %d, out=%q", code, out)
}
if code, out, errb := runOut("{digits(3)}", "--no-shipped-data"); code != 0 || len(out) != 3 {
t.Errorf("--no-shipped-data alone = %d, %q, %q, want a template reading no data rendered", code, out, errb)
}
if code, _, errb := runOut("{/sv_SE.person}", "--no-shipped-data"); code != 2 || !strings.HasSuffix(errb, "no data is loaded\ntry 'fejkdata --help'\n") {
t.Errorf("--no-shipped-data alone = %d, %q, want a reference to nothing refused", code, errb)
}
}
func TestRunRepeatOfZeroWritesNoRecord(t *testing.T) {
dir := recordDir(t)
for _, c := range []struct {
args []string
want string
}{
{nil, ""},
{[]string{"--format", "json"}, "[]\n"},
{[]string{"--format", "ndjson"}, ""},
{[]string{"--format", "csv"}, "first,last\n"},
{[]string{"--format", "sql"}, ""},
} {
if code, out, errb := runOut("{/users}", append(c.args, "-n", "0", "-d", dir)...); code != 0 || out != c.want {
t.Errorf("run(-n 0 %v) = %d, %q, %q, want %q", c.args, code, out, errb, c.want)
}
}
if code, _, _ := runOut("{/nope}", "-n", "0", "--format", "json", "-d", dir); code != 1 {
t.Errorf("run(-n 0) of a reference to nothing = %d, want 1", code)
}
}
func TestRunRepeatIsBounded(t *testing.T) {
for _, args := range [][]string{
{"--repeat", "9223372036854775807"},
{"--repeat", "1048577"},
{"-n", "99999999999"},
} {
code, out, errb := runOut("{/sv_SE.word}", args...)
if code != 2 || out != "" || !strings.Contains(errb, "0..1048576") {
t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb)
}
}
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil {
t.Fatal(err)
}
code, out, errb := runOut("{/x}", "--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "")
if code != 0 || len(out) != 1048576 {
t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb)
}
}
func TestRunIgnoresAFlagWithNoEffect(t *testing.T) {
dir := recordDir(t)
for _, c := range []struct {
stdin string
base, ignored []string
}{
{"", []string{"--list"}, []string{"-n", "3"}},
{"", []string{"--list"}, []string{"--separator", ","}},
{"", []string{"--list"}, []string{"--format", "json"}},
{"", []string{"--list"}, []string{"--table", "t"}},
{"{/users}", nil, []string{"--table", "t"}},
{"{/users}", []string{"--format", "json"}, []string{"--table", "t"}},
} {
args := append([]string{"--seed", "1", "--data-path", dir}, c.base...)
_, want, _ := runOut(c.stdin, args...)
code, got, errb := runOut(c.stdin, append(args, c.ignored...)...)
if code != 0 || want == "" || got != want {
t.Errorf("run(%v + %v) = %d, %q, %q; want what %v alone prints, %q", c.base, c.ignored, code, got, errb, c.base, want)
}
}
}
func TestRunReadsABareNumberAsAFormatString(t *testing.T) {
if code, out, errb := runOut("42\n", "-n", "3"); code != 0 || out != "42\n42\n42\n" {
t.Errorf("echo 42 | fejkdata -n 3 = %d, %q, %q, want 42 on three lines", code, out, errb)
}
}
func TestRunRendersAnyJSONValue(t *testing.T) {
for stdin, want := range map[string]string{"42": "42", "42\n": "42\n", "true\n": "true\n", " 7 ": " 7 ", "null\n": "", " null ": ""} {
if code, out, errb := runOut(stdin); code != 0 || out != want {
t.Errorf("run(%q) = %d, %q, %q, want %q", stdin, code, out, errb, want)
}
}
}
func TestRunUnknownFlagIsNamedByRune(t *testing.T) {
code, _, errb := runOut("{/sv_SE.word}", "-ä")
if code != 2 || !strings.Contains(errb, "unknown flag -ä") {
t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb)
}
}
func TestRunEmptyDataPathIsNamed(t *testing.T) {
code, _, errb := runOut("{/sv_SE.word}", "--data-path=")
if code != 1 || !strings.Contains(errb, "empty") {
t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb)
}
}
func recordDir(t *testing.T) string {
t.Helper()
dir := t.TempDir()
content := `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"]}`
if err := os.WriteFile(filepath.Join(dir, "users.json"), []byte(content), 0o644); err != nil {
t.Fatal(err)
}
return dir
}
func TestRunRecordJSON(t *testing.T) {
code, out, errb := runOut("{/users}", "--seed", "1", "--format", "json", "--repeat", "2", "--data-path", recordDir(t))
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
var arr []map[string]string
if err := json.Unmarshal([]byte(out), &arr); err != nil {
t.Fatalf("json output is not one JSON array: %v\n%q", err, out)
}
if len(arr) != 2 || len(arr[0]) != 2 || arr[0]["first"] == "" || arr[0]["last"] == "" {
t.Fatalf("json output = %q, want an array of two records with first and last columns", out)
}
}
func TestRunRecordNDJSON(t *testing.T) {
code, out, errb := runOut("{/users}", "--seed", "1", "--format", "ndjson", "--repeat", "2", "--data-path", recordDir(t))
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
if len(lines) != 2 {
t.Fatalf("ndjson output = %d lines %q, want one object per line", len(lines), out)
}
for _, line := range lines {
var m map[string]string
if err := json.Unmarshal([]byte(line), &m); err != nil || len(m) != 2 {
t.Fatalf("ndjson line %q is not one object: %v", line, err)
}
}
}
func TestRunRecordCSVRoundTrips(t *testing.T) {
code, out, errb := runOut("{/users}\r\n", "--seed", "1", "--format", "csv", "--repeat", "3", "--data-path", recordDir(t))
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
if len(lines) != 4 {
t.Fatalf("csv output = %d lines %q, want a header and 3 rows", len(lines), out)
}
if lines[0] != "first,last" {
t.Fatalf("csv header = %q, want first,last", lines[0])
}
r, err := csv.NewReader(strings.NewReader(out)).ReadAll()
if err != nil {
t.Fatalf("csv output is not valid CSV: %v", err)
}
if len(r) != 4 || len(r[0]) != 2 {
t.Fatalf("csv parsed to %v, want 4 records of 2 fields", r)
}
}
func TestRunRecordSQL(t *testing.T) {
code, out, errb := runOut("{/users}", "--seed", "1", "--format", "sql", "--repeat", "2", "--table", "people", "--data-path", recordDir(t))
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) {
t.Fatalf("sql output = %q, want two INSERT INTO people statements", out)
}
}
func TestRunRecordDefaultTable(t *testing.T) {
code, out, errb := runOut("{/users}", "--seed", "1", "--format", "sql", "--data-path", recordDir(t))
if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
t.Fatalf("default table = %d, %q, want the reference's last segment; stderr %q", code, out, errb)
}
}
func TestRunRecordOfAQuotedLoneReference(t *testing.T) {
code, out, errb := runOut(`"{/users}"`, "--seed", "1", "--format", "sql", "--data-path", recordDir(t))
if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
t.Fatalf("a JSON string holding {/users} = %d, %q, %q; want the record users", code, out, errb)
}
code, out, errb = runOut("{/users} ", "--format", "csv", "--data-path", recordDir(t))
if code != 2 || out != "" || !strings.Contains(errb, "whitespace around {/users}") {
t.Errorf("{/users} and a space = %d, %q, %q; want misuse naming the whitespace", code, out, errb)
}
}
func TestRunRecordInlineTemplate(t *testing.T) {
code, out, errb := runOut(`{"format":"{x}","x":["a","b"]}`, "--seed", "1", "--format", "json")
if code != 0 {
t.Fatalf("inline record = %d, stderr=%q", code, errb)
}
var arr []map[string]string
if err := json.Unmarshal([]byte(out), &arr); err != nil || len(arr) != 1 || (arr[0]["x"] != "a" && arr[0]["x"] != "b") {
t.Fatalf("inline record json = %q, want one record with a column x (err %v)", out, err)
}
}
func TestRunRecordMisuse(t *testing.T) {
for _, c := range []struct {
args []string
want string
}{
{[]string{"--format", "yaml"}, "--format takes csv, json, ndjson, sql or text"},
{[]string{"--format", "sql", "--table", ""}, "--table names the INSERT target"},
{[]string{"--format", "csv", "--separator", ";"}, `--separator joins text renders; --format csv always separates fields with ","`},
{[]string{"--format", "json", "--separator", ","}, "--separator joins text renders"},
{[]string{"--format", "ndjson", "--separator", ","}, "--separator joins text renders"},
{[]string{"--format", "sql", "--separator", ","}, "--separator joins text renders"},
} {
code, out, errb := runOut("{/users}", c.args...)
if code != 2 || out != "" || !strings.Contains(errb, c.want) {
t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
}
}
}
func TestRunRecordOnABareValueIsRuntimeError(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "word.json"), []byte(`["a","b"]`), 0o644); err != nil {
t.Fatal(err)
}
code, _, errb := runOut("{/word}", "--format", "json", "--data-path", dir)
if code != 1 {
t.Fatalf("record on a choice = %d, want exit 1 (runtime error), stderr %q", code, errb)
}
}
func TestRunRecordRejectsATopLevelRepeat(t *testing.T) {
dir := t.TempDir()
content := `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`
if err := os.WriteFile(filepath.Join(dir, "rep.json"), []byte(content), 0o644); err != nil {
t.Fatal(err)
}
code, _, errb := runOut("{/rep}", "--format", "json", "--data-path", dir)
if code != 1 || !strings.Contains(errb, "carries repeat 3") {
t.Errorf("record on a repeating path = %d, %q; want exit 1 naming the repeat", code, errb)
}
code, _, errb = runOut(content, "--format", "json")
if code != 2 || !strings.Contains(errb, "carries repeat 3") {
t.Errorf("record on a repeating inline template = %d, %q; want exit 2 naming the repeat", code, errb)
}
if code, out, _ := runOut("{/rep}", "--seed", "1", "--data-path", dir); code != 0 || strings.TrimSpace(out) != "x-|x-|x-" {
t.Errorf("text view = %d, %q; want the repeat still composed", code, out)
}
}
func TestRunNumbersFollowTheShell(t *testing.T) {
_, want, _ := runOut("{/sv_SE.word}", "--seed", "7", "--repeat", "3")
code, got, errb := runOut("{/sv_SE.word}", "--seed", "007", "--repeat", "+3")
if code != 0 || got != want {
t.Errorf("run(--seed 007 --repeat +3) = %d, %q, stderr %q; want the same as --seed 7 --repeat 3 %q", code, got, errb, want)
}
}
// TestRunRecordOfAFieldlessPath pins the way out of asking a category of one value
// for columns: the record to write, and the quoting a shell needs to take it. The
// template is well-formed and only the data cannot serve it, so the code is 1.
func TestRunRecordOfAFieldlessPath(t *testing.T) {
code, out, errb := runOut("{/sv_SE.personnummer}", "--format", "csv")
if code != 1 {
t.Fatalf("run(--format csv) of {/sv_SE.personnummer} = %d, want 1", code)
}
for _, want := range []string{`{"format":"","personnummer":"{/sv_SE.personnummer}"}`, "single quotes"} {
if !strings.Contains(errb, want) {
t.Errorf("stderr = %q, want it to name %s", errb, want)
}
}
if out != "" {
t.Errorf("stdout = %q, want nothing", out)
}
}
func TestADataPathCategoryFailingAtFirstReachExits1(t *testing.T) {
dir := t.TempDir()
for name, data := range map[string]string{
".fejkdata.json": `{"index": {"bad": {"paths": [""]}}}`,
"bad.json": `"{/nope}"`,
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0o644); err != nil {
t.Fatal(err)
}
}
for _, args := range [][]string{{"-d", dir}, {"-d", dir, "--format", "json"}} {
if code, _, errb := runOut("{/bad}", args...); code != 1 || !strings.Contains(errb, "{/nope}") {
t.Errorf("%v on a category failing to load = %d, %q; want exit 1 naming the reference", args, code, errb)
}
}
}
+52
View File
@@ -0,0 +1,52 @@
package cli
import (
"testing"
"github.com/larvit/fejkdata/internal/grammar"
)
// grammarLoneReference is the path a format reads when the grammar parses it as one path
// read of one reference from the root, and nothing else.
func grammarLoneReference(text string) (string, bool) {
toks, err := grammar.ParseFormat(text)
if err != nil || len(toks) != 1 || toks[0].Kind != grammar.PathRead || len(toks[0].Arms) != 1 {
return "", false
}
sigil, path, err := grammar.RefShape(toks[0].Arms[0])
if err != nil || sigil != "/" {
return "", false
}
return path, true
}
func TestLoneReferenceAgreesWithTheGrammar(t *testing.T) {
for _, text := range []string{
" {/x}",
"x",
"{.x}",
"{/.x}",
"{//x}",
"{/a...b}",
"{/a..b}",
"{/a..}",
"{/a.[x]}",
"{/a.}",
"{/a|/b}",
"{/a}{/b}",
"{/misc.currency[US Dollar (Next day)]}",
"{/sv_SE.person}",
"{/t[a as b].c}",
"{/t[f(x)]}",
"{/x as p}",
"{/x(1)}",
"{/x}}",
"{/}",
} {
path, lone := loneReference(text)
want, wantLone := grammarLoneReference(text)
if lone != wantLone || path != want {
t.Errorf("loneReference(%q) = %q, %v; the grammar reads %q, %v", text, path, lone, want, wantLone)
}
}
}
+5 -505
View File
@@ -1,512 +1,12 @@
// Command fejkdata prints fake values from the shipped data and any directories // Command fejkdata renders the template on its stdin with fake values from every shipped
// layered over it. // module; package cli documents it.
//
// fejkdata sv_SE.person # a full person
// fejkdata sv_SE.person.last # just the surname
// fejkdata --data-path ./mydata sv_SE.person # layer custom data; the last dir wins
// fejkdata --seed 42 sv_SE.address
package main package main
import ( import (
"bufio"
"errors"
"fmt"
"io"
"os" "os"
"runtime/debug"
"sort"
"strconv"
"strings"
"gitea.larvit.se/larvit/fejkdata" "github.com/larvit/fejkdata/cli"
"github.com/larvit/fejkdata/data"
) )
const usage = `Usage: fejkdata [flags] <path|template> func main() { os.Exit(cli.Main(data.Modules()...)) }
<path> a category, or a dotted path into one (person, person.last);
a table's row by key or name: 'misc.territory[SE]', 'misc.territory[Sweden].tld'
<template> a format string or JSON value to render inline, e.g.
'name: {/sv_SE.person.last}' or '{"format":"{x}","x":["bosse","lina"]}'
A layout inside a template is single-quoted, so quote the whole argument with " to
keep it: "{date(1990-01-01,2010-12-31,'2006-01-02')}".
An argument containing a { token, or a JSON object, array or string, is a
template; any other argument is a path (a path never contains a brace or a quote,
and a bracket only as a [selector] after a table's name). Templates reach the
data by reference from the root — {/sv_SE.person.last} — whether the data is
shipped or layered with --data-path. An argument carrying a closing brace or a
quote but no valid JSON names neither.
With --format json, ndjson, csv or sql the argument must name a record — a
template whose fields are its columns — and the rows are written as one JSON
array, one JSON object per line, one CSV row (after a header), or one INSERT.
-d, --data-path D a data directory to layer over the shipped data (repeatable; last wins on a clash)
--format F output form: text (default), json, ndjson, csv or sql
-h, --help print this help, then exit
--list list the paths the data offers, then exit
--no-shipped-data load only the --data-path directories
-n, --repeat N render the value N times, 1..1048576 (default 1)
-s, --seed N seed for reproducible output
--separator S string between repeated values (default newline)
--table T the INSERT target for --format sql (default: the path's last segment, or records for an inline template)
--version print the version, then exit
Flags may come before or after <path|template>; -- ends the flags. A short flag's
value attaches or follows (-n3, -n 3); short flags bundle (-hn 3).
`
type invocation struct {
dirs []string
format string
formatSet bool
help bool
list bool
noShipped bool
paths []string
repeat int
repeatSet bool
seed uint64
seeded bool
separator string
separatorSet bool
table string
tableSet bool
version bool
}
type flagDef struct {
long string
short string
value bool
set func(*invocation, string) error
}
var flagDefs = []flagDef{
{"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }},
{"format", "", true, func(in *invocation, v string) error { in.format, in.formatSet = v, true; return nil }},
{"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }},
{"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }},
{"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }},
{"repeat", "n", true, func(in *invocation, v string) error {
n, err := strconv.Atoi(v)
if err != nil || n < 1 || n > fejkdata.MaxRepeat {
return fmt.Errorf("--repeat needs an integer in 1..%d, got %q", fejkdata.MaxRepeat, v)
}
in.repeat, in.repeatSet = n, true
return nil
}},
{"seed", "s", true, func(in *invocation, v string) error {
n, err := strconv.ParseUint(v, 10, 64)
if err != nil {
return fmt.Errorf("--seed needs an unsigned integer, got %q", v)
}
in.seed, in.seeded = n, true
return nil
}},
{"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }},
{"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }},
{"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }},
}
func flagByLong(name string) *flagDef {
for i := range flagDefs {
if flagDefs[i].long == name {
return &flagDefs[i]
}
}
return nil
}
func flagByShort(name string) *flagDef {
for i := range flagDefs {
if flagDefs[i].short == name {
return &flagDefs[i]
}
}
return nil
}
// flagArg is one flag as written: its definition and the value attached to it,
// if any.
type flagArg struct {
def *flagDef
value string
inline bool
}
// splitFlags reads one argv element as flags: --name or --name=value, or -abc
// where each letter is a short flag and the first that takes a value takes the
// rest of the element.
func splitFlags(arg string) ([]flagArg, error) {
if strings.HasPrefix(arg, "--") {
name, value, inline := strings.Cut(arg[2:], "=")
def := flagByLong(name)
if def == nil {
return nil, fmt.Errorf("unknown flag --%s", name)
}
return []flagArg{{def, value, inline}}, nil
}
if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil {
return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name)
}
var flags []flagArg
letters := arg[1:]
for i, r := range letters {
letter := string(r)
def := flagByShort(letter)
if def == nil {
return nil, fmt.Errorf("unknown flag -%s", letter)
}
if !def.value {
flags = append(flags, flagArg{def: def})
continue
}
rest := letters[i+len(letter):]
if strings.HasPrefix(rest, "=") {
return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s",
letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:])
}
return append(flags, flagArg{def, rest, rest != ""}), nil
}
return flags, nil
}
func parseArgs(argv []string) (invocation, error) {
in := invocation{repeat: 1, separator: "\n", format: "text"}
for i := 0; i < len(argv); i++ {
arg := argv[i]
if arg == "--" {
in.paths = append(in.paths, argv[i+1:]...)
return in, nil
}
if len(arg) < 2 || arg[0] != '-' {
in.paths = append(in.paths, arg)
continue
}
flags, err := splitFlags(arg)
if err != nil {
return in, err
}
for _, fl := range flags {
if !fl.def.value {
if fl.inline {
return in, fmt.Errorf("--%s takes no value", fl.def.long)
}
_ = fl.def.set(&in, "")
continue
}
value := fl.value
if !fl.inline {
if i++; i >= len(argv) {
return in, fmt.Errorf("--%s needs a value", fl.def.long)
}
value = argv[i]
}
if err := fl.def.set(&in, value); err != nil {
return in, err
}
}
}
return in, nil
}
// recordFormat is one way to write a record out: the line each record renders,
// the header that precedes the first one, and the open/close frame plus the
// between-record separator a document form needs.
type recordFormat struct {
header func(*fejkdata.Record) string
line func(r *fejkdata.Record, table string) string
open string
close string
sep string
}
// recordFormats is every --format that writes records. json frames the records
// as one array document; ndjson is the same column, one object per line.
var recordFormats = map[string]recordFormat{
"csv": {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }, sep: "\n"},
"json": {line: jsonLine, open: "[", close: "]", sep: ",\n"},
"ndjson": {line: jsonLine, sep: "\n"},
"sql": {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }, sep: "\n"},
}
func jsonLine(r *fejkdata.Record, _ string) string { return r.JSON() }
// writesRecords reports whether the format writes records rather than plain text.
func (in invocation) writesRecords() bool {
return in.format != "text"
}
// formatNames lists the --format values, text included, for the misuse error.
func formatNames() string {
names := []string{"text"}
for name := range recordFormats {
names = append(names, name)
}
sort.Strings(names)
return strings.Join(names[:len(names)-1], ", ") + " or " + names[len(names)-1]
}
// checkFlags rejects a flag value or combination that cannot run.
func (in invocation) checkFlags() error {
if _, ok := recordFormats[in.format]; !ok && in.format != "text" {
return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format)
}
if in.tableSet && in.table == "" {
return errors.New("--table names the INSERT target, so it cannot be empty")
}
if in.tableSet && in.format != "sql" {
return errors.New("--table names the INSERT target, so it needs --format sql")
}
if in.writesRecords() && in.separatorSet {
return errors.New("--separator joins text values, so it has no effect with --format " + in.format)
}
return nil
}
// check rejects a flag combination or an argument that cannot run, and reports
// what the argument names, so its shape is settled before any data is read.
func (in invocation) check() (argKind, error) {
if err := in.checkFlags(); err != nil {
return argPath, err
}
if in.list && len(in.paths) > 0 {
return argPath, errors.New("--list takes no path")
}
if in.list && (in.repeatSet || in.separatorSet || in.formatSet || in.tableSet) {
return argPath, errors.New("--list takes no --repeat, --separator, --format or --table")
}
if in.list {
return argPath, nil
}
if len(in.paths) != 1 {
return argPath, fmt.Errorf("expected one path or template, got %d%s", len(in.paths), templateSplit(in.paths))
}
return classify(in.paths[0])
}
func (in invocation) options() []fejkdata.Option {
var opts []fejkdata.Option
if in.noShipped {
opts = append(opts, fejkdata.WithoutShippedData())
}
for _, dir := range in.dirs {
opts = append(opts, fejkdata.WithDataPath(dir))
}
if in.seeded {
opts = append(opts, fejkdata.WithSeed(in.seed))
}
return opts
}
// write streams the argument's renders to w, repeat of them joined by the
// separator and ended by a newline. A value that renders once renders every time,
// so a render failure comes before anything is written; a write failure surfaces
// from Flush, bufio keeping the first one.
func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) error {
if in.writesRecords() {
return in.writeRecords(f, kind, w)
}
draw, err := in.textDraw(f, kind, in.paths[0])
if err != nil {
return err
}
out := bufio.NewWriter(w)
for i := 0; i < in.repeat; i++ {
v, err := draw()
if err != nil {
return err
}
if i > 0 {
out.WriteString(in.separator)
}
out.WriteString(v)
}
out.WriteString("\n")
return out.Flush()
}
// textDraw builds what one text render yields: the value, from a path or an
// inline template.
func (in invocation) textDraw(f *fejkdata.Generator, kind argKind, arg string) (func() (string, error), error) {
if kind != argTemplate {
return func() (string, error) { return f.Fake(arg) }, nil
}
t, err := f.NewTemplate(arg)
if err != nil {
return nil, templateError{err}
}
return func() (string, error) { return t.Fake(), nil }, nil
}
// recordStream builds the record drawer for the argument, plus the INSERT table
// a sql format names.
func (in invocation) recordStream(f *fejkdata.Generator, kind argKind, arg string) (func() (*fejkdata.Record, error), string, error) {
record := func() (*fejkdata.Record, error) { return f.FakeRecord(arg) }
if kind == argTemplate {
t, err := f.NewRecordTemplate(arg)
if err != nil {
return nil, "", templateError{err}
}
record = func() (*fejkdata.Record, error) { return t.Fake(), nil }
}
table := in.table
if table == "" {
table = defaultTable(arg, kind)
}
return record, table, nil
}
// writeRecords streams a record per line in the chosen format, framing a document
// form with its open/close brackets and a header preceding the first record.
func (in invocation) writeRecords(f *fejkdata.Generator, kind argKind, w io.Writer) error {
record, table, err := in.recordStream(f, kind, in.paths[0])
if err != nil {
return err
}
format := recordFormats[in.format]
out := bufio.NewWriter(w)
if format.open != "" {
out.WriteString(format.open)
out.WriteByte('\n')
}
for i := 0; i < in.repeat; i++ {
r, err := record()
if err != nil {
return err
}
if i == 0 && format.header != nil {
out.WriteString(format.header(r))
out.WriteByte('\n')
}
if i > 0 {
out.WriteString(format.sep)
}
out.WriteString(format.line(r, table))
}
if format.close != "" {
out.WriteByte('\n')
out.WriteString(format.close)
}
out.WriteByte('\n')
return out.Flush()
}
// defaultTable names the INSERT target when --table is absent: the path's last
// segment, or "records" for an inline template that sits in no folder.
func defaultTable(arg string, kind argKind) string {
if kind == argTemplate {
return "records"
}
var names strings.Builder // the path with its [selectors] cut out
for depth, i := 0, 0; i < len(arg); i++ {
switch {
case arg[i] == '[':
depth++
case arg[i] == ']':
depth--
case depth == 0:
names.WriteByte(arg[i])
}
}
segments := strings.Split(names.String(), ".")
return segments[len(segments)-1]
}
// templateError marks a render failure that is the argument's own fault — an
// inline template that does not compile. run reports it as misuse (exit 2, with a
// pointer to --help), unlike an unknown path, which is a runtime error (exit 1).
type templateError struct{ error }
func (e templateError) Unwrap() error { return e.error }
type argKind int
const (
argPath argKind = iota
argTemplate
)
// classify reads what a positional argument names by its shape (see fejkdata.IsTemplate).
func classify(arg string) (argKind, error) {
inline, err := fejkdata.IsTemplate(arg)
if inline {
return argTemplate, nil
}
return argPath, err
}
func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) }
// run returns the exit code: 0 ok, 1 runtime error, 2 misuse.
func run(args []string, stdout, stderr io.Writer) int {
in, err := parseArgs(args)
if err != nil {
return misuse(stderr, err)
}
if in.help {
fmt.Fprint(stdout, usage)
return 0
}
if in.version {
fmt.Fprintln(stdout, "fejkdata "+buildVersion())
return 0
}
kind, err := in.check()
if err != nil {
return misuse(stderr, err)
}
f, err := fejkdata.New(in.options()...)
if errors.Is(err, fejkdata.ErrNoData) {
return misuse(stderr, errors.New("--no-shipped-data needs at least one --data-path"))
}
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
if in.list {
for _, p := range f.List() {
fmt.Fprintln(stdout, p)
}
return 0
}
if err := in.write(f, kind, stdout); err != nil {
var te templateError
if errors.As(err, &te) {
return misuse(stderr, te.error)
}
fmt.Fprintln(stderr, err)
if errors.Is(err, fejkdata.ErrNoColumns) {
fmt.Fprintln(stderr, "wrap that JSON in single quotes, which keep a shell from expanding its braces")
}
return 1
}
return 0
}
// templateSplit names the cure for a template a shell split on its spaces, which is
// what several arguments carrying a token mean.
func templateSplit(paths []string) string {
for _, p := range paths {
if strings.Contains(p, "{") {
return `; a template's spaces split the argument, so wrap the whole argument in "…", or in '…' where it carries double quotes`
}
}
return ""
}
func misuse(stderr io.Writer, err error) int {
// A library error already names the program, so the prefix is not doubled.
fmt.Fprintf(stderr, "fejkdata: %s\ntry 'fejkdata --help'\n", strings.TrimPrefix(err.Error(), "fejkdata: "))
return 2
}
// buildVersion is the module version go install stamps into the binary, or "devel".
func buildVersion() string {
if info, ok := debug.ReadBuildInfo(); ok && info.Main.Version != "" {
return info.Main.Version
}
return "devel"
}
+30 -624
View File
@@ -2,640 +2,46 @@ package main
import ( import (
"bytes" "bytes"
"encoding/csv"
"encoding/json"
"os" "os"
"path/filepath" "os/exec"
"regexp"
"strings" "strings"
"testing" "testing"
) )
const ( func TestMain(m *testing.M) {
svSE = "../../data/sv_SE" if os.Getenv("FEJKDATA_RUN_MAIN") == "1" {
enUS = "../../data/en_US" main()
misc = "../../data/misc" }
) os.Exit(m.Run())
func runOut(args ...string) (int, string, string) {
var out, errb bytes.Buffer
code := run(args, &out, &errb)
return code, out.String(), errb.String()
} }
func TestRunOutputsValue(t *testing.T) { func TestMainRendersTheShippedData(t *testing.T) {
code, out, errb := runOut("--data-path", svSE, "person") cmd := exec.Command(os.Args[0], "--seed", "1")
if code != 0 { cmd.Env = append(os.Environ(), "FEJKDATA_RUN_MAIN=1")
t.Fatalf("run = %d, stderr=%q", code, errb) cmd.Stdin = strings.NewReader("{/misc.territory[SE].alpha2} {/en_US.person.last} {/sv_SE.address.locality}")
} var stderr bytes.Buffer
if strings.TrimSpace(out) == "" { cmd.Stderr = &stderr
t.Fatalf("empty output, stderr=%q", errb) out, err := cmd.Output()
} if fields := strings.Fields(string(out)); err != nil || len(fields) < 3 || fields[0] != "SE" {
if !strings.HasSuffix(out, "\n") { t.Errorf("fejkdata = %v, %q, %q, want SE followed by a last name and a locality", err, out, stderr.String())
t.Errorf("output should end with newline, got %q", out)
} }
} }
func TestRunDotPath(t *testing.T) { func TestMainWithoutASeedDrawsANewSeedEachRun(t *testing.T) {
code, full, _ := runOut("--seed", "7", "--data-path", svSE, "person") var outs [2]string
if code != 0 { for i := range outs {
t.Fatalf("person run = %d", code) cmd := exec.Command(os.Args[0])
} cmd.Env = append(os.Environ(), "FEJKDATA_RUN_MAIN=1")
code, last, errb := runOut("--seed", "7", "--data-path", svSE, "person.last") cmd.Stdin = strings.NewReader("{uuid()}")
if code != 0 { var stderr bytes.Buffer
t.Fatalf("person.last run = %d, stderr=%q", code, errb) cmd.Stderr = &stderr
} out, err := cmd.Output()
if strings.TrimSpace(last) == "" { if err != nil || len(out) == 0 {
t.Fatal("empty person.last output") t.Fatalf("fejkdata without --seed = %v, %q, %q", err, out, stderr.String())
}
if last == full {
t.Errorf("person.last %q should differ from person %q", last, full)
}
}
func TestRunSeedSpellings(t *testing.T) {
_, want, _ := runOut("--seed", "42", "--data-path", svSE, "address")
for _, args := range [][]string{
{"--seed=42", "--data-path", svSE, "address"},
{"-s", "42", "-d", svSE, "address"},
{"--data-path", svSE, "address", "--seed", "42"},
} {
code, got, errb := runOut(args...)
if code != 0 {
t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
}
if got != want {
t.Errorf("run(%v) = %q, want %q", args, got, want)
} }
} outs[i] = string(out)
} }
if outs[0] == outs[1] {
func TestRunShortFlagValues(t *testing.T) { t.Errorf("two runs without --seed both printed %q, want a new seed each run", outs[0])
_, want, _ := runOut("--seed", "42", "--data-path", svSE, "address")
for _, args := range [][]string{
{"-s42", "-d", svSE, "address"},
{"-s", "42", "-d" + svSE, "address"},
{"-d", svSE, "address", "-s42"},
} {
code, got, errb := runOut(args...)
if code != 0 {
t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
}
if got != want {
t.Errorf("run(%v) = %q, want %q", args, got, want)
}
}
_, three, _ := runOut("-s", "1", "-n3", "-d", svSE, "word")
if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 {
t.Errorf("-n3 gave %d lines: %q", len(lines), three)
}
for _, c := range []struct {
args []string
want string
}{
{[]string{"-s=42", "-d", svSE, "address"}, "-s42"},
{[]string{"-s=42", "-d", svSE, "address"}, "--seed=42"},
{[]string{"-d=" + svSE, "address"}, "--data-path="},
{[]string{"-nd", "3", "-d", svSE, "word"}, `--repeat needs an integer in 1..1048576, got "d"`},
{[]string{"--seed=", "-d", svSE, "word"}, `--seed needs an unsigned integer, got ""`},
} {
code, out, errb := runOut(c.args...)
if code != 2 || out != "" {
t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out)
}
if !strings.Contains(errb, c.want) {
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
}
}
code, out, _ := runOut("-hd", svSE)
if code != 0 || !strings.Contains(out, "Usage") {
t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out)
}
}
func TestRunDoubleDashEndsFlags(t *testing.T) {
code, out, errb := runOut("--data-path", svSE, "--", "person")
if code != 0 || strings.TrimSpace(out) == "" {
t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
}
code, _, errb = runOut("--data-path", svSE, "--", "--list")
if code != 1 || !strings.Contains(errb, "--list") {
t.Errorf("after --, --list should be a path: code %d, stderr %q", code, errb)
}
}
func TestRunSingleDashLongIsRejected(t *testing.T) {
for _, c := range []struct {
args []string
want string
}{
{[]string{"-seed", "42", "-d", svSE, "person"}, "use --seed"},
{[]string{"-seed=42", "-d", svSE, "person"}, "use --seed"},
{[]string{"-data-path", svSE, "person"}, "use --data-path"},
{[]string{"-list", "-d", svSE}, "use --list"},
{[]string{"--nope", "-d", svSE, "person"}, "unknown flag --nope"},
{[]string{"-x", "-d", svSE, "person"}, "unknown flag -x"},
} {
code, _, errb := runOut(c.args...)
if code != 2 {
t.Errorf("run(%v) = %d, want 2", c.args, code)
}
if !strings.Contains(errb, c.want) {
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
}
}
}
func TestRunFlagValues(t *testing.T) {
for _, c := range []struct {
args []string
want string
}{
{[]string{"--seed", "-d", svSE, "person"}, `--seed needs an unsigned integer, got "-d"`},
{[]string{"-d", svSE, "person", "--seed"}, "--seed needs a value"},
{[]string{"--seed", "x", "-d", svSE, "person"}, "--seed"},
{[]string{"--list=1", "-d", svSE}, "--list takes no value"},
{[]string{"--repeat", "0", "-d", svSE, "person"}, "--repeat"},
{[]string{"-n", "-1", "-d", svSE, "person"}, "--repeat"},
} {
code, _, errb := runOut(c.args...)
if code != 2 {
t.Errorf("run(%v) = %d, want 2", c.args, code)
}
if !strings.Contains(errb, c.want) {
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
}
}
}
func TestRunRepeat(t *testing.T) {
code, out, errb := runOut("--seed", "1", "--repeat", "3", "--data-path", svSE, "word")
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
if len(lines) != 3 {
t.Errorf("repeat=3 gave %d lines: %q", len(lines), out)
}
_, short, _ := runOut("--seed", "1", "-n", "3", "-d", svSE, "word")
if short != out {
t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out)
}
}
func TestRunSeparator(t *testing.T) {
code, out, errb := runOut("--repeat", "3", "--separator", ",", "--data-path", svSE, "word")
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
if n := strings.Count(out, "\n"); n != 1 {
t.Errorf("want one trailing newline, got %d: %q", n, out)
}
if !strings.Contains(out, ",") {
t.Errorf("values should be comma-joined: %q", out)
}
}
func TestRunRepeatAdvancesRNG(t *testing.T) {
code, out, errb := runOut("--seed", "1", "--repeat", "5", "--data-path", svSE, "person")
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
uniq := map[string]bool{}
for _, l := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
uniq[l] = true
}
if len(uniq) < 2 {
t.Errorf("repeat should vary output, all identical: %q", out)
}
}
func TestRunMisuse(t *testing.T) {
for _, args := range [][]string{{}, {"--data-path", svSE}, {"-d", svSE, "person", "word"}, {"-d", svSE, "--list", "person"}, {"--no-shipped-data", "sv_SE.person"}} {
code, out, errb := runOut(args...)
if code != 2 {
t.Errorf("run(%v) = %d, want 2", args, code)
}
if !strings.Contains(errb, "try 'fejkdata --help'") {
t.Errorf("run(%v) stderr = %q, want a pointer to --help", args, errb)
}
if out != "" {
t.Errorf("run(%v) stdout = %q, want nothing", args, out)
}
}
// A template a shell split names both quotings, since which one cures it turns
// on whether the argument carries a double-quoted JSON.
_, _, errb := runOut("{date(1990-01-01,2010-12-31,", "January", "2,", "2006)}")
for _, want := range []string{`wrap the whole argument in "…"`, `or in '…'`} {
if !strings.Contains(errb, want) {
t.Errorf("a split template stderr = %q, want it to name %s", errb, want)
}
}
}
func TestRunMultipleDataPaths(t *testing.T) {
code, out, errb := runOut("-d", enUS, "--data-path", svSE, "person")
if code != 0 {
t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb)
}
if strings.TrimSpace(out) == "" {
t.Fatal("empty output for multi-dir run")
}
}
func TestRunUnknownCategoryFails(t *testing.T) {
code, _, errb := runOut("--data-path", svSE, "nope")
if code != 1 {
t.Fatalf("run = %d, want 1", code)
}
if !strings.Contains(errb, "nope") {
t.Errorf("stderr %q should name the unknown category", errb)
}
}
func TestRunList(t *testing.T) {
code, out, errb := runOut("--data-path", svSE, "--list")
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
for _, want := range []string{"person", "person.last", "address"} {
if !strings.Contains(out, want) {
t.Errorf("list output missing %q:\n%s", want, out)
}
}
}
func TestRunHelpOnStdout(t *testing.T) {
for _, h := range []string{"-h", "--help"} {
code, out, errb := runOut(h)
if code != 0 {
t.Errorf("%s = %d, want 0", h, code)
}
if !strings.Contains(out, "Usage") || errb != "" {
t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb)
}
}
}
func TestRunVersion(t *testing.T) {
code, out, errb := runOut("--version")
if code != 0 {
t.Fatalf("--version = %d, stderr=%q", code, errb)
}
version, ok := strings.CutPrefix(out, "fejkdata ")
if !ok || strings.TrimSpace(version) == "" {
t.Errorf("--version = %q, want the command name and a version on stdout", out)
}
}
func TestRunMissingDirFails(t *testing.T) {
code, _, errb := runOut("--data-path", "../../data/nope", "person")
if code != 1 {
t.Fatalf("run = %d, want 1", code)
}
if errb == "" {
t.Error("want an error message on stderr")
}
}
func TestRunShippedDataByDefault(t *testing.T) {
code, out, errb := runOut("--seed", "1", "sv_SE.person")
if code != 0 || strings.TrimSpace(out) == "" {
t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
}
code, list, _ := runOut("--list")
if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") {
t.Errorf("--list without --data-path = %d, %q", code, list)
}
code, _, errb = runOut("person")
if code != 1 || !strings.Contains(errb, "person") {
t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb)
}
}
func TestClassify(t *testing.T) {
for arg, want := range map[string]argKind{"sv_SE.person": argPath, "name: {x}": argTemplate} {
if got, err := classify(arg); err != nil || got != want {
t.Errorf("classify(%q) = %v, %v; want %v", arg, got, err, want)
}
}
if _, err := classify("[abc]"); err == nil || !strings.Contains(err.Error(), `"["`) || !strings.Contains(err.Error(), "JSON") {
t.Errorf("classify([abc]) = %v; want it rejected naming the leading bracket", err)
}
if got, err := classify("geo.SE.municipality[St. Louis].name"); err != nil || got != argPath {
t.Errorf("classify(a selecting path) = %v, %v; want a path", got, err)
}
}
func TestRunSelectsATableRow(t *testing.T) {
dir := t.TempDir()
for name, content := range map[string]string{
"region.json": `{"format":"{name}","rows":"region.tsv","key":"code","name":"name"}`,
"region.tsv": "code\tname\n01\tStockholms län\n12\tSkåne län\n",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
if code, out, errb := runOut("--no-shipped-data", "-d", dir, "region[12]"); code != 0 || out != "Skåne län\n" {
t.Fatalf("region[12] = %d, %q, stderr=%q", code, out, errb)
}
if code, out, _ := runOut("--no-shipped-data", "-d", dir, "region[Skåne län].code"); code != 0 || out != "12\n" {
t.Fatalf("region[Skåne län].code = %d, %q", code, out)
}
if code, out, _ := runOut("--no-shipped-data", "-d", dir, "--format", "sql", "region[12]"); code != 0 || out != `INSERT INTO "region" ("code", "name") VALUES ('12', 'Skåne län');`+"\n" {
t.Fatalf("--format sql region[12] = %d, %q, want the table named without its selector", code, out)
}
if code, _, errb := runOut("--no-shipped-data", "-d", dir, "region[99]"); code != 1 || !strings.Contains(errb, `"99"`) {
t.Fatalf("region[99] = %d, stderr=%q, want a runtime error naming the row", code, errb)
}
if code, _, errb := runOut("--no-shipped-data", "-d", dir, "[Skåne län]"); code != 2 || !strings.Contains(errb, "JSON") {
t.Fatalf("[Skåne län] = %d, stderr=%q, want misuse: a leading bracket that is no JSON names nothing", code, errb)
}
if code, out, _ := runOut("--seed", "1", "--format", "csv", "misc.territory[SE]"); code != 0 || !strings.HasPrefix(out, "alpha2,") || !strings.Contains(out, "\nSE,SWE,") {
t.Fatalf("--format csv misc.territory[SE] = %d, %q", code, out)
}
}
func TestUsageReferencesResolve(t *testing.T) {
for _, token := range regexp.MustCompile(`\{/[^}]+\}`).FindAllString(usage, -1) {
path := token[2 : len(token)-1]
code, out, errb := runOut("--seed", "1", path)
if code != 0 || strings.TrimSpace(out) == "" {
t.Errorf("usage advertises %s: run %s = %d, %q, stderr %q", token, path, code, out, errb)
}
}
}
func TestRunShapeMisuseBeforeLoad(t *testing.T) {
code, _, errb := runOut("--no-shipped-data", "[abc]")
if code != 2 || !strings.Contains(errb, "[abc]") || strings.Contains(errb, "--data-path") {
t.Fatalf("shape misuse with no data = %d, %q; want the shape error before any load", code, errb)
}
}
func TestRunInlineTemplate(t *testing.T) {
code, out, errb := runOut("--seed", "1", "name: {/sv_SE.person.last}")
if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
t.Fatalf("inline format string = %d, %q, stderr %q", code, out, errb)
}
code, out, errb = runOut("--seed", "1", `{"format":"name: {x}","x":["bosse","lina"]}`)
if code != 0 || (out != "name: bosse\n" && out != "name: lina\n") {
t.Fatalf("inline JSON template = %d, %q, want one name, stderr %q", code, out, errb)
}
code, out, errb = runOut("--seed", "1", `"name: {/sv_SE.person.last}"`)
if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
t.Fatalf("inline JSON string = %d, %q, stderr %q", code, out, errb)
}
code, out, errb = runOut("--seed", "1", "-n", "2", `{digits(1)}`)
if code != 0 || len(strings.Split(strings.TrimRight(out, "\n"), "\n")) != 2 {
t.Fatalf("inline template with --repeat = %d, %q, stderr %q", code, out, errb)
}
}
func TestRunTemplateMisuse(t *testing.T) {
for arg, want := range map[string]string{
"{bad": "unterminated",
"[red,green]": `starts with "["`,
`{"format":"x"}`: "is a string",
"name: {/no.such.path}": "no entry",
"{/sv_SE.person}": "write sv_SE.person",
"x[1]y": `"]"`,
` ["a","b"] `: "may not be padded",
} {
code, out, errb := runOut("--seed", "1", arg)
if code != 2 || out != "" || !strings.Contains(errb, "try 'fejkdata --help'") || !strings.Contains(errb, want) {
t.Errorf("run(%q) = %d, %q, %q; want misuse naming %q and --help", arg, code, out, errb, want)
}
if strings.Contains(errb, "fejkdata: fejkdata:") {
t.Errorf("run(%q) doubled the program prefix: %q", arg, errb)
}
}
}
func TestRunNoShippedData(t *testing.T) {
code, list, errb := runOut("--no-shipped-data", "-d", misc, "--list")
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
if strings.Contains(list, "sv_SE") || !strings.Contains(list, "uuid\n") {
t.Errorf("--no-shipped-data --list = %q, want only the given dir", list)
}
code, out, _ := runOut("--no-shipped-data", "-d", misc, "-s", "3", "uuid")
if code != 0 || strings.TrimSpace(out) == "" {
t.Errorf("run = %d, out=%q", code, out)
}
code, _, errb = runOut("--no-shipped-data", "sv_SE.person")
if code != 2 || !strings.Contains(errb, "--no-shipped-data needs at least one --data-path") {
t.Errorf("--no-shipped-data alone = %d, %q, want misuse naming --data-path", code, errb)
}
}
func TestRunRepeatIsBounded(t *testing.T) {
for _, args := range [][]string{
{"--repeat", "9223372036854775807", "sv_SE.word"},
{"--repeat", "1048577", "sv_SE.word"},
{"-n", "99999999999", "sv_SE.word"},
} {
code, out, errb := runOut(args...)
if code != 2 || out != "" || !strings.Contains(errb, "1..1048576") {
t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb)
}
}
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil {
t.Fatal(err)
}
code, out, errb := runOut("--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "", "x")
if code != 0 || len(out) != 1048576+1 {
t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb)
}
}
func TestRunListTakesNoRepeatOrSeparator(t *testing.T) {
for _, args := range [][]string{{"--list", "-n", "3"}, {"--list", "--separator", ","}} {
code, _, errb := runOut(args...)
if code != 2 || !strings.Contains(errb, "--list takes no") {
t.Errorf("run(%v) = %d, %q, want misuse", args, code, errb)
}
}
}
func TestRunUnknownFlagIsNamedByRune(t *testing.T) {
code, _, errb := runOut("-ä", "sv_SE.word")
if code != 2 || !strings.Contains(errb, "unknown flag -ä") {
t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb)
}
}
func TestRunEmptyDataPathIsNamed(t *testing.T) {
code, _, errb := runOut("--data-path=", "sv_SE.word")
if code != 1 || !strings.Contains(errb, "empty") {
t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb)
}
}
func recordDir(t *testing.T) string {
t.Helper()
dir := t.TempDir()
content := `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"]}`
if err := os.WriteFile(filepath.Join(dir, "users.json"), []byte(content), 0o644); err != nil {
t.Fatal(err)
}
return dir
}
func TestRunRecordJSON(t *testing.T) {
code, out, errb := runOut("--seed", "1", "--format", "json", "--repeat", "2", "--data-path", recordDir(t), "users")
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
var arr []map[string]string
if err := json.Unmarshal([]byte(out), &arr); err != nil {
t.Fatalf("json output is not one JSON array: %v\n%q", err, out)
}
if len(arr) != 2 || len(arr[0]) != 2 || arr[0]["first"] == "" || arr[0]["last"] == "" {
t.Fatalf("json output = %q, want an array of two records with first and last columns", out)
}
}
func TestRunRecordNDJSON(t *testing.T) {
code, out, errb := runOut("--seed", "1", "--format", "ndjson", "--repeat", "2", "--data-path", recordDir(t), "users")
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
if len(lines) != 2 {
t.Fatalf("ndjson output = %d lines %q, want one object per line", len(lines), out)
}
for _, line := range lines {
var m map[string]string
if err := json.Unmarshal([]byte(line), &m); err != nil || len(m) != 2 {
t.Fatalf("ndjson line %q is not one object: %v", line, err)
}
}
}
func TestRunRecordCSVRoundTrips(t *testing.T) {
code, out, errb := runOut("--seed", "1", "--format", "csv", "--repeat", "3", "--data-path", recordDir(t), "users")
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
if len(lines) != 4 {
t.Fatalf("csv output = %d lines %q, want a header and 3 rows", len(lines), out)
}
if lines[0] != "first,last" {
t.Fatalf("csv header = %q, want first,last", lines[0])
}
r, err := csv.NewReader(strings.NewReader(out)).ReadAll()
if err != nil {
t.Fatalf("csv output is not valid CSV: %v", err)
}
if len(r) != 4 || len(r[0]) != 2 {
t.Fatalf("csv parsed to %v, want 4 records of 2 fields", r)
}
}
func TestRunRecordSQL(t *testing.T) {
code, out, errb := runOut("--seed", "1", "--format", "sql", "--repeat", "2", "--table", "people", "--data-path", recordDir(t), "users")
if code != 0 {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) {
t.Fatalf("sql output = %q, want two INSERT INTO people statements", out)
}
}
func TestRunRecordDefaultTable(t *testing.T) {
code, out, errb := runOut("--seed", "1", "--format", "sql", "--data-path", recordDir(t), "users")
if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
t.Fatalf("default table = %d, %q, want the path's last segment; stderr %q", code, out, errb)
}
}
func TestRunRecordInlineTemplate(t *testing.T) {
code, out, errb := runOut("--seed", "1", "--format", "json", `{"format":"{x}","x":["a","b"]}`)
if code != 0 {
t.Fatalf("inline record = %d, stderr=%q", code, errb)
}
var arr []map[string]string
if err := json.Unmarshal([]byte(out), &arr); err != nil || len(arr) != 1 || (arr[0]["x"] != "a" && arr[0]["x"] != "b") {
t.Fatalf("inline record json = %q, want one record with a column x (err %v)", out, err)
}
}
func TestRunRecordMisuse(t *testing.T) {
for _, c := range []struct {
args []string
want string
}{
{[]string{"--format", "yaml", "users"}, "--format takes csv, json, ndjson, sql or text"},
{[]string{"--format", "json", "--separator", ",", "users"}, "--separator joins text values"},
{[]string{"--table", "t", "users"}, "--table names the INSERT target"},
{[]string{"--format", "json", "--table", "t", "users"}, "--table names the INSERT target"},
{[]string{"--format", "sql", "--table", "", "users"}, "--table names the INSERT target"},
} {
code, out, errb := runOut(c.args...)
if code != 2 || out != "" || !strings.Contains(errb, c.want) {
t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
}
}
}
func TestRunRecordOnABareValueIsRuntimeError(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "word.json"), []byte(`["a","b"]`), 0o644); err != nil {
t.Fatal(err)
}
code, _, errb := runOut("--format", "json", "--data-path", dir, "word")
if code != 1 {
t.Fatalf("record on a choice = %d, want exit 1 (runtime error), stderr %q", code, errb)
}
}
func TestRunRecordRejectsATopLevelRepeat(t *testing.T) {
dir := t.TempDir()
content := `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`
if err := os.WriteFile(filepath.Join(dir, "rep.json"), []byte(content), 0o644); err != nil {
t.Fatal(err)
}
code, _, errb := runOut("--format", "json", "--data-path", dir, "rep")
if code != 1 || !strings.Contains(errb, "carries repeat 3") {
t.Errorf("record on a repeating path = %d, %q; want exit 1 naming the repeat", code, errb)
}
code, _, errb = runOut("--format", "json", content)
if code != 2 || !strings.Contains(errb, "carries repeat 3") {
t.Errorf("record on a repeating inline template = %d, %q; want exit 2 naming the repeat", code, errb)
}
if code, out, _ := runOut("--seed", "1", "--data-path", dir, "rep"); code != 0 || strings.TrimSpace(out) != "x-|x-|x-" {
t.Errorf("text view = %d, %q; want the repeat still composed", code, out)
}
}
func TestRunNumbersFollowTheShell(t *testing.T) {
_, want, _ := runOut("--seed", "7", "--repeat", "3", "sv_SE.word")
code, got, errb := runOut("--seed", "007", "--repeat", "+3", "sv_SE.word")
if code != 0 || got != want {
t.Errorf("run(--seed 007 --repeat +3) = %d, %q, stderr %q; want the same as --seed 7 --repeat 3 %q", code, got, errb, want)
}
}
// TestRunRecordOfAFieldlessPath pins the way out of asking a category of one value
// for columns: the record to write, and the quoting a shell needs to take it. The
// argument is well-formed and only the data cannot serve it, so the code is 1.
func TestRunRecordOfAFieldlessPath(t *testing.T) {
code, out, errb := runOut("--format", "csv", "sv_SE.personnummer")
if code != 1 {
t.Fatalf("run(--format csv sv_SE.personnummer) = %d, want 1", code)
}
for _, want := range []string{`{"format":"","personnummer":"{/sv_SE.personnummer}"}`, "single quotes"} {
if !strings.Contains(errb, want) {
t.Errorf("stderr = %q, want it to name %s", errb, want)
}
}
if out != "" {
t.Errorf("stdout = %q, want nothing", out)
} }
} }
+11 -20
View File
@@ -1,21 +1,5 @@
# Docker-based dev workflow — no local tooling beyond Docker is required.
#
# docker compose run --rm test
# docker compose run --rm vet
# docker compose run --rm build
#
# The full gate — vet, formatting and tests — is `docker build .`, which is what
# CI runs.
#
# Build caches persist in the `gocache` volume, so re-runs are fast and the
# host tree stays clean. Commands that rewrite source (fmt, tidy) keep your
# ownership when run with `--user "$(id -u):$(id -g)"`.
#
# GO_VERSION picks the toolchain image (default: latest stable). Test the lowest
# supported Go with: GO_VERSION=1.22 docker compose run --rm test
x-go: &go x-go: &go
image: golang:${GO_VERSION:-1.26.4} image: golang:${GO_VERSION:-1.27.2}
working_dir: /app working_dir: /app
volumes: volumes:
- .:/app - .:/app
@@ -33,12 +17,16 @@ services:
cover: cover:
<<: *go <<: *go
command: go test -cover ./... command: sh -c "go test -coverpkg=./... -coverprofile=/tmp/cover.out ./... && go tool cover -func=/tmp/cover.out | tail -1"
bench: bench:
<<: *go <<: *go
command: go test -run XXX -bench . -benchmem ./... command: go test -run XXX -bench . -benchmem ./...
generate:
<<: *go
command: go generate ./...
vet: vet:
<<: *go <<: *go
command: go vet ./... command: go vet ./...
@@ -47,6 +35,10 @@ services:
<<: *go <<: *go
command: go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test . command: go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test .
vulncheck:
<<: *go
command: go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan module
build: build:
<<: *go <<: *go
command: go build ./... command: go build ./...
@@ -59,13 +51,12 @@ services:
<<: *go <<: *go
command: go mod tidy command: go mod tidy
# Interactive shell for ad-hoc work: docker compose run --rm dev
dev: dev:
<<: *go <<: *go
command: sh command: sh
data-import: data-import:
image: python:3.14.7-slim image: python:3.14.8-slim
working_dir: /app working_dir: /app
volumes: volumes:
- .:/app - .:/app
+43 -65
View File
@@ -1,14 +1,12 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Rebuild data/geo/SE/*.tsv from SCB (CC0), GeoNames (CC BY 4.0) and Trafikverket NVDB (CC0). """Rebuild data/geo/SE/*.tsv from SCB (CC0), GeoNames (CC BY 4.0) and Trafikverket NVDB (CC0).
TRAFIKVERKET_API_KEY=… data-import/geo-se.py [--key-file FILE] [--cache DIR] [--streets-per-locality N] [--out DIR] TRAFIKVERKET_API_KEY=… data-import/geo-se.py [--key-file FILE] [--cache DIR] [--min-streets N] [--street-share F] [--out DIR]
""" """
import argparse
import collections import collections
import csv import csv
import io import io
import json import json
import math
import os import os
import re import re
import sys import sys
@@ -16,6 +14,7 @@ import urllib.request
import zipfile import zipfile
from pathlib import Path from pathlib import Path
import geo
import source import source
import tsv import tsv
import xlsx import xlsx
@@ -34,11 +33,15 @@ TATORTER = "https://geodata.scb.se/geoserver/stat/wfs?service=WFS&version=2.0.0&
POSTAL_CODES = "https://download.geonames.org/export/zip/SE.zip" POSTAL_CODES = "https://download.geonames.org/export/zip/SE.zip"
NVDB = "https://api.trafikinfo.trafikverket.se/v2/data.json" NVDB = "https://api.trafikinfo.trafikverket.se/v2/data.json"
NVDB_PAGE = 50000 NVDB_PAGE = 50000
OUT = Path(__file__).resolve().parent.parent / "data" / "geo" / "SE"
CACHE = Path(__file__).resolve().parent / "cache"
TIMEZONE = "Europe/Stockholm" TIMEZONE = "Europe/Stockholm"
# This script's reading of the "exceptions in big cities" in docs/research/research-geo-se.md, Postal codes → Structure.
ONE_POSITION = {"Stockholm", "Göteborg", "Malmö"} ONE_POSITION = {"Stockholm", "Göteborg", "Malmö"}
# A tätort's floor: "minst 200 invånare", docs/research/research-geo-se.md, Lantmäteriet → Ortnamn.
UNMATCHED_POPULATION = 200 UNMATCHED_POPULATION = 200
# Not a street: a road number, a ferry route, a cycle path's or road association's label, a placeholder, or two names
# or a description in one.
NOT_A_STREET = re.compile(r"^(Europaväg|Länsväg|Riksväg) \d|^Färjeled |\b(GC|SMF|VSF)\b|^NN$|,")
PARTICLES = {"af", "de", "von"}
def scb_codes(cache): def scb_codes(cache):
@@ -104,36 +107,11 @@ def nvdb_segments(cache, key):
yield name, float(lat), float(lon) yield name, float(lat), float(lon)
class Nearest:
"""Nearest point by an equirectangular distance, over a degree grid."""
def __init__(self, points, cell=0.05):
self.cell = cell
self.grid = collections.defaultdict(list)
for lat, lon, value in points:
self.grid[(int(lat // cell), int(lon // cell))].append((lat, lon, value))
def find(self, lat, lon):
ci, cj = int(lat // self.cell), int(lon // self.cell)
best, best_d = None, math.inf
ring = 0
while ring < 400:
for i in range(ci - ring, ci + ring + 1):
for j in range(cj - ring, cj + ring + 1):
if max(abs(i - ci), abs(j - cj)) != ring:
continue
for plat, plon, value in self.grid.get((i, j), ()):
d = (plat - lat) ** 2 + ((plon - lon) * math.cos(math.radians(lat))) ** 2
if d < best_d:
best, best_d = value, d
if best is not None and math.sqrt(best_d) < ring * self.cell * math.cos(math.radians(lat)):
return best
ring += 1
return best
def street_delivery(name, codes): def street_delivery(name, codes):
"""The codes delivered to a street: the digit after the postort's own prefix says box, company or reply.""" """The codes delivered to a street. A postort's own prefix is three digits where one three-digit prefix starts at
least half its codes, else two. Outside ONE_POSITION's cities, the digit after that prefix is 0 or 1 for boxes and 8
for reply mail; inside them, a second digit 0 marks boxes. The box and reply-mail digits are per
docs/research/research-geo-se.md, Postal codes → Structure."""
if name in ONE_POSITION: if name in ONE_POSITION:
return [c for c in codes if c[1] != "0"] return [c for c in codes if c[1] != "0"]
largest = collections.Counter(c[:3] for c in codes).most_common(1)[0][1] largest = collections.Counter(c[:3] for c in codes).most_common(1)[0][1]
@@ -163,6 +141,8 @@ def population_of(name, municipality, tatorter, municipalities, population):
def well_cased(name): def well_cased(name):
"""Whether each word of name is capitalised: GeoNames also spells some postorter, such as STOCKHOLM and örebro, in
capitals or lower case."""
return all(part[:1].isupper() and (len(part) == 1 or not part.isupper()) for part in re.split(r"[ -]", name)) return all(part[:1].isupper() and (len(part) == 1 or not part.isupper()) for part in re.split(r"[ -]", name))
@@ -176,58 +156,56 @@ def localities(codes, tatorter, municipalities, population):
municipality, method = municipality_of(name, rows, tatorter, municipalities) municipality, method = municipality_of(name, rows, tatorter, municipalities)
how[method] += 1 how[method] += 1
kept = street_delivery(name, [r["code"].replace(" ", "") for r in rows]) kept = street_delivery(name, [r["code"].replace(" ", "") for r in rows])
with_point = [r for r in rows if r["lat"] is not None] point = geo.centroid(rows)
if municipality is None or not kept or not with_point or not well_cased(name): if municipality is None or not kept or point is None or not well_cased(name):
continue continue
lat = sum(r["lat"] for r in with_point) / len(with_point) lat, lon = point
lon = sum(r["lon"] for r in with_point) / len(with_point)
out[name] = {"name": name, "municipality": municipality, "population": population_of(name, municipality, tatorter, municipalities, population), "lat": f"{lat:.4f}", "lon": f"{lon:.4f}", "codes": kept} out[name] = {"name": name, "municipality": municipality, "population": population_of(name, municipality, tatorter, municipalities, population), "lat": f"{lat:.4f}", "lon": f"{lon:.4f}", "codes": kept}
print(f"municipality by {dict(how)}; {len(by_locality) - len(out)} postorter dropped", file=sys.stderr) print(f"municipality by {dict(how)}; {len(by_locality) - len(out)} postorter dropped before streets", file=sys.stderr)
return out return out
def streets(segments, codes, localities, per_locality): def street_name(name):
# A word of four capitals or fewer is an acronym or a Roman numeral: IOGT-vägen, Karl XII gatan.
opening = name[0].isupper() or name.split()[0] in PARTICLES
return opening and not NOT_A_STREET.search(name) and not any(len(w) >= 5 and w.isupper() for w in name.split())
def streets(segments, codes, localities, share, minimum):
"""The names with most segments per locality, each segment at its nearest code centroid.""" """The names with most segments per locality, each segment at its nearest code centroid."""
nearest = Nearest((r["lat"], r["lon"], r["locality"]) for r in codes if r["lat"] is not None and r["locality"] in localities) nearest = geo.Nearest((r["lat"], r["lon"], r["locality"]) for r in codes if r["lat"] is not None and r["locality"] in localities)
count = collections.Counter() count = collections.Counter()
for name, lat, lon in segments: for name, lat, lon in segments:
if name[0].isalpha(): if street_name(name):
count[(nearest.find(lat, lon), name)] += 1 count[(nearest.find(lat, lon), name)] += 1
of = collections.defaultdict(list) return geo.top_streets(count, share, minimum, "segments")
for (locality, name), n in count.items():
of[locality].append((n, name))
return {locality: [{"name": name, "locality": locality, "segments": n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:per_locality]] for locality, named in of.items()}
def main(): def main():
p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) a = geo.parser(__doc__, "SE", 1.0, key_file={"help": "file holding the Trafikverket API key; TRAFIKVERKET_API_KEY otherwise"}).parse_args()
p.add_argument("--cache", default=str(CACHE))
p.add_argument("--key-file", help="file holding the Trafikverket API key; TRAFIKVERKET_API_KEY otherwise")
p.add_argument("--out", default=str(OUT))
p.add_argument("--streets-per-locality", type=int, default=10)
a = p.parse_args()
key = Path(a.key_file).read_text().strip() if a.key_file else os.environ.get("TRAFIKVERKET_API_KEY") key = Path(a.key_file).read_text().strip() if a.key_file else os.environ.get("TRAFIKVERKET_API_KEY")
if not key: if not key:
sys.exit("set TRAFIKVERKET_API_KEY or pass --key-file") sys.exit("set TRAFIKVERKET_API_KEY or pass --key-file")
cache, out = Path(a.cache), Path(a.out) cache, out = geo.directories(a)
cache.mkdir(parents=True, exist_ok=True)
out.mkdir(parents=True, exist_ok=True)
regions, municipalities = scb_codes(cache) region_names, municipality_names = scb_codes(cache)
population = scb_population(cache) population = scb_population(cache)
codes = geonames(cache) codes = geonames(cache)
places = localities(codes, scb_tatorter(cache), municipalities, population) places = localities(codes, scb_tatorter(cache), municipality_names, population)
named = streets(nvdb_segments(cache, key), codes, places, a.streets_per_locality) named = streets(nvdb_segments(cache, key), codes, places, a.street_share, a.min_streets)
places = {name: l for name, l in places.items() if name in named} regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in region_names.items()}
empty = sorted(m for m in municipalities if not any(l["municipality"] == m for l in places.values())) municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipality_names.items()}
if empty: kept, dropped = geo.fence(regions, municipalities, places, named, {k: l["codes"] for k, l in places.items()})
sys.exit(f"municipalities without a locality: {empty}") geo.log_dropped_localities(dropped["locality"])
if dropped["municipality"] or dropped["region"]:
sys.exit(f"municipalities without a locality: {sorted(dropped['municipality'])}; regions without a municipality: {sorted(dropped['region'])}")
places = kept["locality"]
tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [{"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in sorted(regions.items())]) tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [{"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in sorted(municipalities.items())]) tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(kept["municipality"].items())])
tsv.write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())]) tsv.write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
tsv.write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in places.values() for c in l["codes"]), key=lambda r: r["code"])) tsv.write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in places.values() for c in l["codes"]), key=lambda r: r["code"]))
tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(named) for s in named[locality]]) tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(places) for s in named[locality]])
if __name__ == "__main__": if __name__ == "__main__":
+76 -41
View File
@@ -1,19 +1,21 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Rebuild data/geo/US/*.tsv from the Census Bureau's Gazetteer, population estimates, ZCTA relationships and TIGER/Line files (public domain). """Rebuild data/geo/US/*.tsv from the Census Bureau's Gazetteer, population estimates, ZCTA relationships and TIGER/Line files (public domain), and Wikidata's state capitals (CC0).
data-import/geo-us.py [--cache DIR] [--min-population N] [--streets-per-locality N] [--out DIR] data-import/geo-us.py [--cache DIR] [--min-streets N] [--place-share F] [--street-share F] [--out DIR]
""" """
import argparse
import collections import collections
import concurrent.futures import concurrent.futures
import csv import csv
import io import io
import math
import random
import re import re
import struct import struct
import sys import sys
import urllib.parse
import zipfile import zipfile
from pathlib import Path
import geo
import source import source
import tsv import tsv
@@ -24,14 +26,20 @@ COUNTIES = POPULATION.format("counties/totals/co-est2025-alldata.csv")
PLACES = POPULATION.format("cities/totals/sub-est2025.csv") PLACES = POPULATION.format("cities/totals/sub-est2025.csv")
ZCTA_PLACE = "https://www2.census.gov/geo/docs/maps-data/data/rel2020/zcta520/tab20_zcta520_place20_natl.txt" ZCTA_PLACE = "https://www2.census.gov/geo/docs/maps-data/data/rel2020/zcta520/tab20_zcta520_place20_natl.txt"
TIGER = "https://www2.census.gov/geo/tiger/TIGER2025/{0}/tl_2025_{1}_{2}.zip" TIGER = "https://www2.census.gov/geo/tiger/TIGER2025/{0}/tl_2025_{1}_{2}.zip"
OUT = Path(__file__).resolve().parent.parent / "data" / "geo" / "US"
CACHE = Path(__file__).resolve().parent / "cache"
ESTIMATE = "POPESTIMATE2025" ESTIMATE = "POPESTIMATE2025"
CDP = "57" CDP = "57"
# Each state's capital from Wikidata, with the capital's GNIS ids, which the Gazetteer carries as ANSICODE.
CAPITALS = "https://query.wikidata.org/sparql?" + urllib.parse.urlencode({"query": """
SELECT ?iso ?capitalLabel ?gnis WHERE {
?state wdt:P31 wd:Q35657; wdt:P300 ?iso; wdt:P36 ?capital.
OPTIONAL { ?capital wdt:P590 ?gnis. }
SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
}"""})
SHIPPED_POPULATION = 100000
HIGHWAY = re.compile(r"\b(I- |Hwy |Highway |Loop |Rte |Route |Rd )\d") HIGHWAY = re.compile(r"\b(I- |Hwy |Highway |Loop |Rte |Route |Rd )\d")
SUFFIX = re.compile(r" (city and borough|city|town|village|borough|municipality|comunidad|zona urbana|metropolitan government|metro government|consolidated government|unified government|urban county|corporation|plantation)( \(balance\))?$") SUFFIX = re.compile(r" (city and borough|city|town|village|borough|municipality|comunidad|zona urbana|metropolitan government|metro government|consolidated government|unified government|urban county|corporation|plantation)( \(balance\))?$")
# Places whose Census name is a merged government's; the postal city is what an address carries. # Places whose Census name is not the postal city an address carries.
NAMES = {"1303440": "Athens", "1304204": "Augusta", "1349008": "Macon", "2148006": "Louisville", "3011397": "Butte", "4732742": "Hartsville", "4752006": "Nashville"} NAMES = {"0665042": "Ventura", "1303440": "Athens", "1304204": "Augusta", "1349008": "Macon", "1608830": "Boise", "2146027": "Lexington", "2148006": "Louisville", "3011397": "Butte", "4732742": "Hartsville", "4744382": "Lynchburg", "4752006": "Nashville"}
TIMEZONES = { TIMEZONES = {
"AK": "America/Anchorage", "AL": "America/Chicago", "AR": "America/Chicago", "AZ": "America/Phoenix", "AK": "America/Anchorage", "AL": "America/Chicago", "AR": "America/Chicago", "AZ": "America/Phoenix",
"CA": "America/Los_Angeles", "CO": "America/Denver", "CT": "America/New_York", "DC": "America/New_York", "CA": "America/Los_Angeles", "CO": "America/Denver", "CT": "America/New_York", "DC": "America/New_York",
@@ -107,8 +115,8 @@ def place_name(geoid, name):
return stripped return stripped
def localities(cache, min_population, counties): def localities(cache, counties):
"""Each shipped place with its county, population and centroid.""" """Each incorporated place with its county, population and centroid."""
place_population, county_part = {}, collections.defaultdict(list) place_population, county_part = {}, collections.defaultdict(list)
for r in csv_rows(cache, PLACES, "sub-est2025.csv"): for r in csv_rows(cache, PLACES, "sub-est2025.csv"):
if r["SUMLEV"] == "162": if r["SUMLEV"] == "162":
@@ -118,18 +126,47 @@ def localities(cache, min_population, counties):
out = {} out = {}
for r in gazetteer(cache, "place"): for r in gazetteer(cache, "place"):
geoid, population = r["GEOID"], place_population.get(r["GEOID"], 0) geoid, population = r["GEOID"], place_population.get(r["GEOID"], 0)
if r["FUNCSTAT"] not in ("A", "F", "N") or r["LSAD"] == CDP or population < min_population or not county_part.get(geoid): if r["FUNCSTAT"] not in ("A", "B", "F", "N") or r["LSAD"] == CDP or not county_part.get(geoid):
continue continue
county = max(county_part[geoid])[1] county = max(county_part[geoid])[1]
if county not in counties: if county not in counties:
print(f"{geoid} {r['NAME']}: county {county} unknown, dropped", file=sys.stderr) print(f"{geoid} {r['NAME']}: county {county} unknown, dropped", file=sys.stderr)
continue continue
out[geoid] = {"code": geoid, "name": place_name(geoid, r["NAME"]), "municipality": county, "population": population, "lat": r["INTPTLAT"], "lon": r["INTPTLONG"]} out[geoid] = {"code": geoid, "gnis": int(r["ANSICODE"]), "name": place_name(geoid, r["NAME"]), "municipality": county, "population": population, "lat": r["INTPTLAT"], "lon": r["INTPTLONG"]}
return out return out
def postal_codes(cache, localities): def capitals(cache, places, counties):
"""Each ZCTA whose largest part inside an incorporated place lies in a shipped place.""" """The codes of the places that are a state's capital, matched by any of its GNIS ids, else by a name no other place in the state carries."""
by_gnis = {p["gnis"]: code for code, p in places.items()}
named = collections.defaultdict(list)
for code, p in places.items():
named[(counties[p["municipality"]]["region"], p["name"])].append(code)
by_name = {key: codes[0] for key, codes in named.items() if len(codes) == 1}
gnis_of = collections.defaultdict(set)
for r in csv.DictReader(io.StringIO(text(source.fetch(CAPITALS, cache, "capitals.csv", magic=b"iso,", headers={"Accept": "text/csv"})))):
gnis_of[(r["iso"].removeprefix("US-"), r["capitalLabel"])].add(int(r["gnis"] or 0))
out = set()
for (state, name), ids in sorted(gnis_of.items()):
code = next((by_gnis[i] for i in sorted(ids) if i in by_gnis), None) or by_name.get((state, name))
if code:
out.add(code)
else:
print(f"{state} {name}: capital matches no incorporated place by GNIS id or by a name unique in its state", file=sys.stderr)
return out
def drawn(places, share):
"""The share of the places with people, each drawn with odds by its population: Efraimidis and Spirakis's weighted
sampling without replacement. Each place's random number is seeded by its code, so a place added to or dropped from
the sources moves only the places near the cut."""
peopled = [code for code, p in places.items() if p["population"] > 0]
key = {code: math.log(1 - random.Random(code).random()) / places[code]["population"] for code in peopled}
return {code: places[code] for code in sorted(peopled, key=lambda c: (key[c], c), reverse=True)[:round(len(peopled) * share)]}
def zcta_localities(cache, localities):
"""Each ZCTA mapped to the incorporated place holding most of its land inside incorporated places, where localities holds that place."""
parts = {} parts = {}
for r in csv.DictReader(io.StringIO(text(source.fetch(ZCTA_PLACE, cache, "zcta-place.txt"))), delimiter="|"): for r in csv.DictReader(io.StringIO(text(source.fetch(ZCTA_PLACE, cache, "zcta-place.txt"))), delimiter="|"):
if r["GEOID_ZCTA5_20"] and r["GEOID_PLACE_20"] and not r["NAMELSAD_PLACE_20"].endswith(" CDP"): if r["GEOID_ZCTA5_20"] and r["GEOID_PLACE_20"] and not r["NAMELSAD_PLACE_20"].endswith(" CDP"):
@@ -138,7 +175,7 @@ def postal_codes(cache, localities):
return {zcta: place for zcta, place in largest.items() if place in localities} return {zcta: place for zcta, place in largest.items() if place in localities}
def streets(cache, counties, locality_of_zcta, per_locality): def streets(cache, counties, locality_of_zcta, share, minimum):
"""The names with most TIGER address ranges per place, and the address ranges per ZCTA.""" """The names with most TIGER address ranges per place, and the address ranges per ZCTA."""
with concurrent.futures.ThreadPoolExecutor(3) as pool: with concurrent.futures.ThreadPoolExecutor(3) as pool:
list(pool.map(lambda c: (tiger_zip(cache, "addr", c), tiger_zip(cache, "featnames", c)), counties)) list(pool.map(lambda c: (tiger_zip(cache, "addr", c), tiger_zip(cache, "featnames", c)), counties))
@@ -153,23 +190,12 @@ def streets(cache, counties, locality_of_zcta, per_locality):
if r["PAFLAG"] == "P" and r["FULLNAME"] and not HIGHWAY.search(r["FULLNAME"]): if r["PAFLAG"] == "P" and r["FULLNAME"] and not HIGHWAY.search(r["FULLNAME"]):
for z in zips.get(r["TLID"], ()): for z in zips.get(r["TLID"], ()):
count[(locality_of_zcta[z], r["FULLNAME"])] += 1 count[(locality_of_zcta[z], r["FULLNAME"])] += 1
of = collections.defaultdict(list) return geo.top_streets(count, share, minimum, "addresses"), addresses
for (locality, name), n in count.items():
of[locality].append((n, name))
named = {locality: [{"name": name, "locality": locality, "addresses": n} for n, name in sorted(ranked, key=lambda s: (-s[0], s[1]))[:per_locality]] for locality, ranked in of.items()}
return named, addresses
def main(): def main():
p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) a = geo.parser(__doc__, "US", 0.25, place_share={"type": float, "default": 0.1}).parse_args()
p.add_argument("--cache", default=str(CACHE)) cache, out = geo.directories(a)
p.add_argument("--min-population", type=int, default=25000)
p.add_argument("--out", default=str(OUT))
p.add_argument("--streets-per-locality", type=int, default=10)
a = p.parse_args()
cache, out = Path(a.cache), Path(a.out)
cache.mkdir(parents=True, exist_ok=True)
out.mkdir(parents=True, exist_ok=True)
state_population = {r["STATE"]: r[ESTIMATE] for r in csv_rows(cache, STATES, "nst-est2025.csv") if r["SUMLEV"] == "040"} state_population = {r["STATE"]: r[ESTIMATE] for r in csv_rows(cache, STATES, "nst-est2025.csv") if r["SUMLEV"] == "040"}
regions = {r["USPS"]: {"abbr": r["USPS"], "code": r["GEOID"], "name": r["NAME"], "population": state_population[r["GEOID"]], "timezone": TIMEZONES[r["USPS"]]} for r in gazetteer(cache, "state")} regions = {r["USPS"]: {"abbr": r["USPS"], "code": r["GEOID"], "name": r["NAME"], "population": state_population[r["GEOID"]], "timezone": TIMEZONES[r["USPS"]]} for r in gazetteer(cache, "state")}
@@ -182,20 +208,29 @@ def main():
counties[r["GEOID"]] = {"code": r["GEOID"], "name": r["NAME"], "region": r["USPS"], "population": county_population[r["GEOID"]]} counties[r["GEOID"]] = {"code": r["GEOID"], "name": r["NAME"], "region": r["USPS"], "population": county_population[r["GEOID"]]}
print(f"counties without a population estimate, dropped: {dict(unestimated)}", file=sys.stderr) print(f"counties without a population estimate, dropped: {dict(unestimated)}", file=sys.stderr)
places = localities(cache, a.min_population, counties) places = localities(cache, counties)
locality_of_zcta = postal_codes(cache, places) owner_of_zcta = zcta_localities(cache, places)
named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.streets_per_locality) capital_codes = capitals(cache, places, counties)
for geoid in [l for l in places if l not in named]: owners = {p: places[p] for p in set(owner_of_zcta.values())}
print(f"{geoid} {places[geoid]['name']}: no streets, dropped", file=sys.stderr) for code in sorted(capital_codes - set(owners)):
del places[geoid] print(f"{code} {places[code]['name']}: capital owns no ZCTA, dropped", file=sys.stderr)
kept_counties = {l["municipality"] for l in places.values()} places = drawn(owners, a.place_share) | {code: p for code, p in owners.items() if code in capital_codes or p["population"] >= SHIPPED_POPULATION}
kept_regions = {counties[c]["region"] for c in kept_counties} locality_of_zcta = {zcta: p for zcta, p in owner_of_zcta.items() if p in places}
named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.street_share, a.min_streets)
zctas = collections.defaultdict(list)
for zcta, locality in sorted(locality_of_zcta.items()):
if addresses[zcta]:
zctas[locality].append(zcta)
kept, dropped = geo.fence(regions, counties, places, named, zctas)
geo.log_dropped_localities(dropped["locality"])
print(f"counties without a place, dropped: {len(dropped['municipality'])}; states without a county, dropped: {sorted(dropped['region'])}", file=sys.stderr)
places = kept["locality"]
tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(regions.items()) if r["abbr"] in kept_regions]) tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(counties.items()) if c["code"] in kept_counties]) tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(kept["municipality"].items())])
tsv.write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())]) tsv.write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], [{"code": z, "locality": l, "addresses": addresses[z]} for z, l in sorted(locality_of_zcta.items()) if addresses[z] and l in places]) tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], sorted(({"code": z, "locality": l, "addresses": addresses[z]} for l in places for z in zctas[l]), key=lambda r: r["code"]))
tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(named) for s in named[locality]]) tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(places) for s in named[locality]])
if __name__ == "__main__": if __name__ == "__main__":
+98
View File
@@ -0,0 +1,98 @@
"""The steps a country's geo import may share with another's."""
import argparse
import collections
import math
import sys
from pathlib import Path
DATA = Path(__file__).resolve().parent.parent / "data" / "geo"
CACHE = Path(__file__).resolve().parent / "cache"
def parser(doc, country, street_share, **own):
"""A parser for the options every country's import takes, plus the script's own: each keyword names one option, with
"_" standing for "-"."""
options = {"cache": {"default": str(CACHE)}, "out": {"default": str(DATA / country)}, "min_streets": {"type": int, "default": 10}, "street_share": {"type": float, "default": street_share}, **own}
p = argparse.ArgumentParser(description=doc.splitlines()[0])
for name in sorted(options):
p.add_argument("--" + name.replace("_", "-"), **options[name])
return p
def directories(args):
"""The cache and out directories of the parsed options, both created."""
cache, out = Path(args.cache), Path(args.out)
cache.mkdir(parents=True, exist_ok=True)
out.mkdir(parents=True, exist_ok=True)
return cache, out
def centroid(rows):
"""The mean lat and lon of the rows with a point, or None where no row has one."""
points = [r for r in rows if r["lat"] is not None]
if not points:
return None
return sum(r["lat"] for r in points) / len(points), sum(r["lon"] for r in points) / len(points)
class Nearest:
"""Nearest point by an equirectangular distance, over a degree grid."""
def __init__(self, points, cell=0.05):
self.cell = cell
self.grid = collections.defaultdict(list)
for lat, lon, value in points:
self.grid[(int(lat // cell), int(lon // cell))].append((lat, lon, value))
def find(self, lat, lon):
ci, cj = int(lat // self.cell), int(lon // self.cell)
best, best_d = None, math.inf
ring = 0
while ring < 400:
for i in range(ci - ring, ci + ring + 1):
for j in range(cj - ring, cj + ring + 1):
if max(abs(i - ci), abs(j - cj)) != ring:
continue
for plat, plon, value in self.grid.get((i, j), ()):
d = (plat - lat) ** 2 + ((plon - lon) * math.cos(math.radians(lat))) ** 2
if d < best_d:
best, best_d = value, d
if best is not None and math.sqrt(best_d) < ring * self.cell * math.cos(math.radians(lat)):
return best
ring += 1
return best
def top_streets(count, share, minimum, column):
"""count maps (locality, street name) to a count. Returns, per locality, rows for its most-counted names, share of
them rounded up and at least minimum, ties broken by name; each row holds its count under column."""
of = collections.defaultdict(list)
for (locality, name), n in count.items():
of[locality].append((n, name))
return {locality: [{"name": name, "locality": locality, column: n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:max(minimum, math.ceil(len(named) * share))]] for locality, named in of.items()}
def fence(regions, municipalities, localities, streets, postal_codes):
"""Meet the loader's fence that every parent row has a child row. Keeps each locality with entries in both streets
and postal_codes, then each municipality and region left with a child.
regions and municipalities map a code to a row; a municipality row names its region under "region". localities,
streets and postal_codes map a locality's key to its row, its streets and its postal codes. A locality row holds
name, municipality, population, lat and lon, plus its country's own keys.
Returns kept, then dropped: each maps "locality", "municipality" and "region" to that table's rows."""
kept = {"locality": {k: l for k, l in localities.items() if streets.get(k) and postal_codes.get(k)}}
municipality_codes = {l["municipality"] for l in kept["locality"].values()}
kept["municipality"] = {c: m for c, m in municipalities.items() if c in municipality_codes}
region_codes = {m["region"] for m in kept["municipality"].values()}
kept["region"] = {c: r for c, r in regions.items() if c in region_codes}
every = {"locality": localities, "municipality": municipalities, "region": regions}
dropped = {table: {k: row for k, row in rows.items() if k not in kept[table]} for table, rows in every.items()}
return kept, dropped
def log_dropped_localities(rows):
"""Logs each locality row as dropped, by its key, plus its name where that differs."""
for key, row in rows.items():
label = key if key == row["name"] else f"{key} {row['name']}"
print(f"{label}: no street or no postal code, dropped", file=sys.stderr)
+1 -1
View File
@@ -23,7 +23,7 @@ CACHE = Path(__file__).resolve().parent / "cache"
COLUMNS = ["idempotent", "method", "safe"] COLUMNS = ["idempotent", "method", "safe"]
CORE = re.compile(r"\[RFC9110, Section 9\.3\.|\[RFC5789, Section 2\]") CORE = re.compile(r"\[RFC9110, Section 9\.3\.|\[RFC5789, Section 2\]")
BOOLEAN = {"yes": "true", "no": "false"} BOOLEAN = {"yes": "true", "no": "false"}
SELECTOR = '[]{}"|' # mirrors inSelector in table.go SELECTOR = '[]{}"|' # mirrors inSelector in internal/grammar/path.go
EXPECTED = 9 EXPECTED = 9
+1 -1
View File
@@ -21,7 +21,7 @@ OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "protocol.tsv"
CACHE = Path(__file__).resolve().parent / "cache" CACHE = Path(__file__).resolve().parent / "cache"
COLUMNS = ["keyword", "name", "number"] COLUMNS = ["keyword", "name", "number"]
SKIP = ("Reserved", "deprecated") SKIP = ("Reserved", "deprecated")
SELECTOR = '[]{}"|' # mirrors inSelector in table.go SELECTOR = '[]{}"|' # mirrors inSelector in internal/grammar/path.go
def rows(text): def rows(text):
+1 -1
View File
@@ -14,7 +14,7 @@ def fetch(source, cache, name, magic=b"", data=None, headers=None):
if path.exists(): if path.exists():
return path.read_bytes() return path.read_bytes()
for attempt in range(1, 6): for attempt in range(1, 6):
req = urllib.request.Request(source, data=data, headers={"User-Agent": "fejkdata data-import", **(headers or {})}) req = urllib.request.Request(source, data=data, headers={"User-Agent": "fejkdata-data-import (https://github.com/larvit/fejkdata)", **(headers or {})})
try: try:
with urllib.request.urlopen(req, timeout=600) as r: with urllib.request.urlopen(req, timeout=600) as r:
body = r.read() body = r.read()
+4 -4
View File
@@ -5,14 +5,14 @@ from pathlib import Path
def write(path, columns, rows): def write(path, columns, rows):
"""Write the rows as a TSV; the table needs two rows, and every cell must be non-empty and free of tabs, newlines and braces.""" """Write the rows as a TSV of a header and at least one row, each cell stripped, non-empty and free of tabs, newlines and braces."""
lines = ["\t".join(columns)] lines = ["\t".join(columns)]
for row in rows: for row in rows:
cells = [str(row[c]) for c in columns] cells = [str(row[c]).strip() for c in columns]
if not all(cells) or any(re.search(r"[\t\n{}]", c) for c in cells): if not all(cells) or any(re.search(r"[\t\n{}]", c) for c in cells):
raise ValueError(f"{path}: a cell is empty or holds a tab, newline or brace: {row}") raise ValueError(f"{path}: a cell is empty or holds a tab, newline or brace: {row}")
lines.append("\t".join(cells)) lines.append("\t".join(cells))
if len(lines) < 3: if len(lines) < 2:
raise ValueError(f"{path}: has {len(lines) - 1} rows below its header; a table is at least two rows") raise ValueError(f"{path}: has no rows below its header; a table holds at least one row")
Path(path).write_text("\n".join(lines) + "\n", encoding="utf-8") Path(path).write_text("\n".join(lines) + "\n", encoding="utf-8")
print(f"{path}: {len(lines) - 1} rows", file=sys.stderr) print(f"{path}: {len(lines) - 1} rows", file=sys.stderr)
+146 -166
View File
@@ -1,195 +1,175 @@
package fejkdata package fejkdata
import ( import (
"encoding/json"
"fmt"
"io/fs"
"os"
"path"
"strings" "strings"
"github.com/larvit/fejkdata/internal/datafiles"
) )
// dataSource is one tree to load: an fs.FS and the directory in it to start from. // loadSources merges sources in order, the last winning a clash. A source whose manifest
// label prefixes file names in errors; onDisk marks path as a directory that must // carries an index leaves its categories unloaded; every other source loads here, with
// exist. // the unloaded categories it needs.
type dataSource struct { // docs/decisions.md#a-source-whose-manifest-carries-an-index-loads-each-category-on-first-reach-any-other-loads-in-new-with-what-it-reads
fsys fs.FS func loadSources(sources []datafiles.Source) (folder, error) {
label string root := folder{children: map[string]node{}}
onDisk bool
path string
root string
}
func (s dataSource) name(p string) string {
if s.label == "" {
return p
}
return path.Join(s.label, p)
}
// loadData loads every source into one namespace tree and returns its root
// children. A directory becomes a folder; each *.json file in it compiles to a node
// keyed by its base name (address.json -> "address"); each subdirectory becomes a
// nested folder, so folders turn into dot-path segments. Sources merge left to right:
// matching folders merge by their children, and any other clash is won by the last
// source loaded. Once merged, linkRefs binds every reference against the
// final tree.
func loadData(sources []dataSource) (map[string]node, error) {
root := map[string]node{}
for _, src := range sources { for _, src := range sources {
if src.onDisk { m, err := src.Manifest()
if src.path == "" {
return nil, fmt.Errorf("a data path is empty")
}
info, err := os.Stat(src.path)
if err != nil {
return nil, fmt.Errorf("data path %s: %w", src.path, err)
}
if !info.IsDir() {
return nil, fmt.Errorf("%s is not a directory", src.path)
}
}
dir := src.root
if dir == "" {
dir = "."
}
g, err := loadDir(src, dir)
if err != nil { if err != nil {
return nil, err return folder{}, err
}
if err := addSource(&root, src, m); err != nil {
return folder{}, err
} }
mergeChildren(root, g.children)
} }
if len(root) == 0 { return root, loadReached(&root, categorySites(&root), nil)
return nil, fmt.Errorf("no .json data found")
}
setTablePaths(root)
if err := linkTables(root); err != nil {
return nil, err
}
if err := linkRefs(root); err != nil {
return nil, err
}
if err := checkNoCycles(root); err != nil {
return nil, err
}
if err := checkScope(treeScope(root)); err != nil {
return nil, err
}
return root, nil
} }
// loadDir compiles one directory into a folder. fs.ReadDir yields entries sorted func addSource(root *folder, src datafiles.Source, m datafiles.Manifest) error {
// by name, so the tree is built deterministically. Empty subdirectories (no JSON src = src.Mounted(m.Namespace)
// anywhere under them) are skipped rather than added as empty namespaces. if m.Index != nil {
func loadDir(src dataSource, dir string) (*folder, error) { placeIndex(root, src, m.Index)
entries, err := fs.ReadDir(src.fsys, dir) return nil
if err != nil {
return nil, fmt.Errorf("%s: %w", src.name(dir), err)
} }
g := &folder{children: map[string]node{}} g := &folder{children: map[string]node{}}
files := &categoryFiles{src: src, dir: dir, tsv: map[string]bool{}} if err := src.Walk(compileInto(func(dir []string) *folder { return madeFolder(g, dir) })); err != nil {
for _, e := range entries {
if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() {
files.tsv[e.Name()] = false
}
}
for _, e := range entries {
if strings.HasPrefix(e.Name(), ".") { // hidden: a checkout or an editor's file, never data
continue
}
full := path.Join(dir, e.Name())
if e.IsDir() {
if err := loadFolder(src, g, full, e.Name()); err != nil {
return nil, err
}
continue
}
if err := loadFile(src, g, full, e.Name(), files); err != nil {
return nil, err
}
}
for name, named := range files.tsv {
if !named && !strings.HasPrefix(name, ".") {
return nil, fmt.Errorf("%s: no category names it in its rows; a table's rows file sits beside a category file naming it", src.name(path.Join(dir, name)))
}
}
return g, nil
}
// categoryFiles is what a category may name beside itself: the rows files of its
// directory, each marked once a category names it.
type categoryFiles struct {
src dataSource
dir string
tsv map[string]bool
}
// rows reads the rows file a category names beside it.
func (c *categoryFiles) rows(name string) (string, error) {
if _, present := c.tsv[name]; !present {
return "", fmt.Errorf("rows names %s, which is not beside it in %s", name, c.src.name(c.dir))
}
c.tsv[name] = true
b, err := fs.ReadFile(c.src.fsys, path.Join(c.dir, name))
if err != nil {
return "", fmt.Errorf("%s: %w", c.src.name(path.Join(c.dir, name)), err)
}
return string(b), nil
}
// loadFolder adds a subdirectory as a nested folder, unless nothing under it is data.
func loadFolder(src dataSource, g *folder, full, name string) error {
child, err := loadDir(src, full)
if err != nil {
return err return err
} }
if len(child.children) == 0 { mergeFolder(root, g)
return nil
}
if err := checkName(name); err != nil {
return fmt.Errorf("%s: folder %w", src.name(full), err)
}
g.children[name] = child
return nil return nil
} }
// loadFile compiles a *.json file into a category named after it; any other file func compileInto(place func(dir []string) *folder) func(datafiles.Category) error {
// is skipped. return func(c datafiles.Category) error {
func loadFile(src dataSource, g *folder, full, file string, files *categoryFiles) error { n, err := compileCategory(c)
if !strings.HasSuffix(file, ".json") { if err != nil {
return err
}
place(c.Folders).put(c.Name, n)
return nil return nil
} }
name := strings.TrimSuffix(file, ".json")
if err := checkName(name); err != nil {
return fmt.Errorf("%s: category %w", src.name(full), err)
}
b, err := fs.ReadFile(src.fsys, full)
if err != nil {
return fmt.Errorf("%s: %w", src.name(full), err)
}
var raw any
if err := json.Unmarshal(b, &raw); err != nil {
return fmt.Errorf("%s: %w", src.name(full), err)
}
n, err := compileCategory(raw, name, files)
if err != nil {
return fmt.Errorf("%s: %w", src.name(full), err)
}
g.children[name] = n
return nil
} }
// mergeChildren overlays src onto dst. Two folders under the same key merge // madeFolder is the folder that dir names under root, made where it is missing or a category.
// recursively (so locales/categories from several paths combine); every other func madeFolder(root *folder, dir []string) *folder {
// key is replaced, making the last-loaded directory win on a conflict. g := root
func mergeChildren(dst, src map[string]node) { for _, seg := range dir {
for k, v := range src { sub, isFolder := g.children[seg].(*folder)
if dg, ok := dst[k].(*folder); ok { if !isFolder {
sub = &folder{children: map[string]node{}}
g.put(seg, sub)
}
g = sub
}
return g
}
func mergeFolder(dst, src *folder) {
for k, v := range src.children {
if dg, ok := dst.children[k].(*folder); ok {
if sg, ok := v.(*folder); ok { if sg, ok := v.(*folder); ok {
mergeChildren(dg.children, sg.children) mergeFolder(dg, sg)
continue continue
} }
} }
dst[k] = v dst.put(k, v)
} }
} }
// nodeSet is the set of nodes one validation pass covers: one load's categories, or a
// single inline node.
type nodeSet func(fn func(label string, n node) error) error
// categorySite is a loaded category, where it sits and its own dot path.
type categorySite struct {
categoryAt
n node
path string
}
// categorySites lists every category under root, folders and names in sorted order.
func categorySites(root *folder) []categorySite {
var out []categorySite
var walk func(dir []string, g *folder)
walk = func(dir []string, g *folder) {
for _, name := range sortedNames(g.children) {
if sub, isFolder := g.children[name].(*folder); isFolder {
walk(append(dir[:len(dir):len(dir)], name), sub)
continue
}
out = append(out, siteIn(dir, g, name))
}
}
walk(nil, root)
return out
}
func siteIn(dir []string, in *folder, name string) categorySite {
return categorySite{categoryAt: categoryAt{dir: dir, in: in, name: name}, n: in.children[name], path: categoryPath(dir, name)}
}
func categoryPath(dir []string, name string) string { return join(strings.Join(dir, "."), name) }
func siteNodes(sites []categorySite) nodeSet {
return func(fn func(label string, n node) error) error {
for _, s := range sites {
if err := eachNode(s.n, s.path, fn); err != nil {
return err
}
}
return nil
}
}
func inlineNodes(n node, label string) nodeSet {
return func(fn func(label string, m node) error) error { return eachNode(n, label, fn) }
}
// pipeline is the one load pipeline, run over one nodeSet; its fields are where
// entry points differ.
type pipeline struct {
nodes nodeSet
resolve func() error
// Set by a caller that proves the columns against their Go types itself.
typedByGo bool
}
// categoryPipeline is the pipeline over the categories of one load, their references
// resolving against root.
func categoryPipeline(sites []categorySite, root map[string]node) pipeline {
return pipeline{
nodes: siteNodes(sites),
resolve: func() error {
if err := linkTables(sites); err != nil {
return err
}
return resolveCategoryTemplates(sites, root)
},
}
}
func (p pipeline) run() error {
if err := p.resolve(); err != nil {
return err
}
if err := checkNoCycles(p.nodes); err != nil {
return err
}
settleRecords(p.nodes)
if !p.typedByGo {
if err := checkColumns(p.nodes); err != nil {
return err
}
}
return checkNodeFences(p.nodes)
}
// checkNodeFences runs the per-node fences over s, each over every node before the next,
// so which of several broken nodes is reported does not depend on the walk. Its walks
// recurse unguarded, so pipeline.run refuses cycles first.
func checkNodeFences(s nodeSet) error {
mem := renderCounts{}
if err := s(func(label string, n node) error { return repeatCheck(label, n, mem) }); err != nil {
return err
}
return s((&valueProof{}).checkDatatype)
}
+27
View File
@@ -0,0 +1,27 @@
{
"index": {
"en_US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
"en_US.color": {"paths":[""]},
"en_US.company": {"paths":["","base","suffix"]},
"en_US.date": {"paths":[""]},
"en_US.email": {"paths":["","domain","local","local.n"]},
"en_US.first-name": {"parent":"sex","paths":["","count","name","sex"]},
"en_US.ip": {"paths":[""]},
"en_US.itin": {"paths":["","area","group","serial"]},
"en_US.last-name": {"paths":["","count","name"]},
"en_US.person": {"paths":["","first","last","prefix","sex"]},
"en_US.phone": {"paths":["","area","exchange","line"]},
"en_US.phone-exchange": {"paths":[""]},
"en_US.price": {"paths":["","amt","cents"]},
"en_US.sentence": {"paths":["","adj","noun","prep","verb"]},
"en_US.sex": {"paths":["","code","first-name","first-name.count","first-name.name","first-name.sex","name","title","title.name","title.sex","title.share"]},
"en_US.ssn": {"paths":["","area","group","serial"]},
"en_US.time": {"paths":[""]},
"en_US.title": {"parent":"sex","paths":["","name","sex","share"]},
"en_US.url": {"paths":["","host","path"]},
"en_US.username": {"paths":["","n"]},
"en_US.version": {"paths":["","n","pre","suffix"]},
"en_US.word": {"paths":[""]}
},
"namespace": "en_US"
}
+6 -6
View File
@@ -1,8 +1,8 @@
{ {
"format": "{street-number} {street}\n{locality}, {region} {postal-code}", "format": "{/geo.US.address as a}{a}",
"street": "{/geo.US.address.street}", "street": "{a.street}",
"street-number": "{/geo.US.address.street-number}", "street-number": "{a.street-number}",
"locality": "{/geo.US.address.locality}", "locality": "{a.locality}",
"region": "{/geo.US.address.region}", "region": "{a.region}",
"postal-code": "{/geo.US.address.postal-code}" "postal-code": "{a.postal-code}"
} }
+19
View File
@@ -0,0 +1,19 @@
// Package enus is fejkdata's en_US locale module, whose categories are en_US.*; its address reads
// geo.US, which package geous holds.
package enus
import (
"embed"
"io/fs"
geous "github.com/larvit/fejkdata/data/geo/US"
)
// FS holds the locale alone, its categories and its manifest; pass Modules unless your own
// data stands in for geo.US.
//
//go:embed *.json *.tsv .fejkdata.json
var FS embed.FS
// Modules returns FS and every module it reads, to pass to fejkdata.WithDataFS.
func Modules() []fs.FS { return append([]fs.FS{FS}, geous.Modules()...) }
+22
View File
@@ -0,0 +1,22 @@
package enus_test
import (
"io/fs"
"testing"
"github.com/larvit/fejkdata"
enus "github.com/larvit/fejkdata/data/en_US"
)
func TestFSLoadsByItsManifest(t *testing.T) {
if _, err := fs.Stat(enus.FS, ".fejkdata.json"); err != nil {
t.Fatalf("FS holds no manifest: %v", err)
}
f, err := fejkdata.New(fejkdata.WithDataFS(enus.Modules()...), fejkdata.WithSeed(1))
if err != nil {
t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
}
if len(f.List()) == 0 {
t.Error("List() is empty, want the module's categories")
}
}
+3 -3
View File
@@ -1,7 +1,7 @@
{ {
"format": "{prefix}{first} {last}", "format": "{prefix}{first} {last}",
"first": "{.first-name.name}", "first": "{.first-name as n}{n.name}",
"last": "{.last-name.name}", "last": "{.last-name.name}",
"prefix": [null, { "format": "{.title.name} ", "weight": 0.1 }], "prefix": [null, { "format": "{n..sex.title.name} ", "weight": 0.1 }],
"sex": "{.sex.name}" "sex": "{n..sex.name}"
} }
+17
View File
@@ -0,0 +1,17 @@
[
{ "format": "{int(2,9)}0{digits(1)}", "weight": 80 },
{ "format": "{int(2,9)}1{int(2,9)}", "weight": 64 },
{ "format": "{int(2,9)}10", "weight": 8 },
{ "format": "{int(2,4)}{int(2,9)}{digits(1)}", "weight": 240 },
{ "format": "{int(6,8)}{int(2,9)}{digits(1)}", "weight": 240 },
{ "format": "5{int(2,4)}{digits(1)}", "weight": 30 },
{ "format": "5{int(6,9)}{digits(1)}", "weight": 40 },
{ "format": "55{int(0,4)}", "weight": 5 },
{ "format": "55{int(6,9)}", "weight": 4 },
{ "format": "9{int(2,4)}{digits(1)}", "weight": 30 },
{ "format": "9{int(8,9)}{digits(1)}", "weight": 20 },
{ "format": "95{int(1,7)}", "weight": 7 },
{ "format": "96{digits(1)}", "weight": 10 },
{ "format": "97{int(0,5)}", "weight": 6 },
{ "format": "97{int(7,9)}", "weight": 3 }
]
+4 -4
View File
@@ -1,15 +1,15 @@
[ [
{ {
"format": "({area}) {exch}-{line}", "format": "({area}) {exchange}-{line}",
"weight": 2, "weight": 2,
"area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"], "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
"exch": "{int(100,999)}", "exchange": "{.phone-exchange}",
"line": "{digits(4)}" "line": "{digits(4)}"
}, },
{ {
"format": "{area}-{exch}-{line}", "format": "{area}-{exchange}-{line}",
"area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"], "area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
"exch": "{int(100,999)}", "exchange": "{.phone-exchange}",
"line": "{digits(4)}" "line": "{digits(4)}"
} }
] ]
+11
View File
@@ -0,0 +1,11 @@
{
"index": {
"geo.SE.address": {"paths":["","locality","postal-code","street","street-number"]},
"geo.SE.locality": {"parent":"municipality","paths":["","lat","lon","municipality","name","population","postal-code","postal-code.code","postal-code.locality","street","street.locality","street.name","street.segments"]},
"geo.SE.municipality": {"parent":"region","paths":["","code","locality","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.locality","locality.street.name","locality.street.segments","name","population","region"]},
"geo.SE.postal-code": {"parent":"locality","paths":["","code","locality"]},
"geo.SE.region": {"paths":["","code","municipality","municipality.code","municipality.locality","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.locality","municipality.locality.street.name","municipality.locality.street.segments","municipality.name","municipality.population","municipality.region","name","population","timezone"]},
"geo.SE.street": {"parent":"locality","paths":["","locality","name","segments"]}
},
"namespace": "geo.SE"
}
+3 -3
View File
@@ -1,6 +1,6 @@
{ {
"format": "{street} {street-number}\n{postal-code} {locality}", "format": "{street} {street-number}\n{postal-code} {locality}",
"street": "{.street.name}", "street": "{.locality as l}{l.street.name}",
"street-number": [ "street-number": [
"{int(1,9)}", "{int(1,9)}",
"{int(10,99)}", "{int(10,99)}",
@@ -9,6 +9,6 @@
{ "format": "{int(10,99)}{upper(1)}", "weight": 0.1 }, { "format": "{int(10,99)}{upper(1)}", "weight": 0.1 },
{ "format": "{int(100,999)}{upper(1)}", "weight": 0.05 } { "format": "{int(100,999)}{upper(1)}", "weight": 0.05 }
], ],
"postal-code": "{.postal-code.code}", "postal-code": "{l.postal-code.code}",
"locality": "{.locality.name}" "locality": "{l.name}"
} }
+15
View File
@@ -0,0 +1,15 @@
// Package geose is fejkdata's Swedish places module, whose categories are geo.SE.*.
package geose
import (
"embed"
"io/fs"
)
// FS holds the module's categories and its manifest.
//
//go:embed *.json *.tsv .fejkdata.json
var FS embed.FS
// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
func Modules() []fs.FS { return []fs.FS{FS} }
+22
View File
@@ -0,0 +1,22 @@
package geose_test
import (
"io/fs"
"testing"
"github.com/larvit/fejkdata"
geose "github.com/larvit/fejkdata/data/geo/SE"
)
func TestFSLoadsByItsManifest(t *testing.T) {
if _, err := fs.Stat(geose.FS, ".fejkdata.json"); err != nil {
t.Fatalf("FS holds no manifest: %v", err)
}
f, err := fejkdata.New(fejkdata.WithDataFS(geose.Modules()...), fejkdata.WithSeed(1))
if err != nil {
t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
}
if len(f.List()) == 0 {
t.Error("List() is empty, want the module's categories")
}
}
-1
View File
@@ -866,7 +866,6 @@ Nynäshamn 0192 16447 58.9034 17.9479
Nyvång 1277 200 56.1333 12.9000 Nyvång 1277 200 56.1333 12.9000
Nyåker 2403 200 63.7500 19.3167 Nyåker 2403 200 63.7500 19.3167
Nälden 2309 986 63.3500 14.2500 Nälden 2309 986 63.3500 14.2500
Nämdö 0120 200 59.1833 18.6833
Nässjö 0682 18639 57.6531 14.6968 Nässjö 0682 18639 57.6531 14.6968
Näsum 1272 1274 56.1667 14.4833 Näsum 1272 1274 56.1667 14.4833
Näsviken 2184 994 61.7667 16.8667 Näsviken 2184 994 61.7667 16.8667
1 name municipality population lat lon
866 Nyvång 1277 200 56.1333 12.9000
867 Nyåker 2403 200 63.7500 19.3167
868 Nälden 2309 986 63.3500 14.2500
Nämdö 0120 200 59.1833 18.6833
869 Nässjö 0682 18639 57.6531 14.6968
870 Näsum 1272 1274 56.1667 14.4833
871 Näsviken 2184 994 61.7667 16.8667
-1
View File
@@ -949,7 +949,6 @@ code locality
130 25 Ingmarsö 130 25 Ingmarsö
130 33 Gällnöby 130 33 Gällnöby
130 34 Skälvik 130 34 Skälvik
130 36 Nämdö
130 37 Stavsnäs 130 37 Stavsnäs
130 38 Runmarö 130 38 Runmarö
130 39 Sandhamn 130 39 Sandhamn
1 code locality
949 130 25 Ingmarsö
950 130 33 Gällnöby
951 130 34 Skälvik
130 36 Nämdö
952 130 37 Stavsnäs
953 130 38 Runmarö
954 130 39 Sandhamn
+159557 -5
View File
File diff suppressed because it is too large Load Diff
+11
View File
@@ -0,0 +1,11 @@
{
"index": {
"geo.US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
"geo.US.locality": {"parent":"municipality","paths":["","code","lat","lon","municipality","name","population","postal-code","postal-code.addresses","postal-code.code","postal-code.locality","street","street.addresses","street.locality","street.name"]},
"geo.US.municipality": {"parent":"region","paths":["","code","locality","locality.code","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.addresses","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.addresses","locality.street.locality","locality.street.name","name","population","region"]},
"geo.US.postal-code": {"parent":"locality","paths":["","addresses","code","locality"]},
"geo.US.region": {"paths":["","abbr","code","municipality","municipality.code","municipality.locality","municipality.locality.code","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.addresses","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.addresses","municipality.locality.street.locality","municipality.locality.street.name","municipality.name","municipality.population","municipality.region","name","population","timezone"]},
"geo.US.street": {"parent":"locality","paths":["","addresses","locality","name"]}
},
"namespace": "geo.US"
}
+4 -4
View File
@@ -1,13 +1,13 @@
{ {
"format": "{street-number} {street}\n{locality}, {region} {postal-code}", "format": "{street-number} {street}\n{locality}, {region} {postal-code}",
"street": "{.street.name}", "street": "{.locality as l}{l.street.name}",
"street-number": [ "street-number": [
"{int(10,99)}", "{int(10,99)}",
{ "format": "{int(100,999)}", "weight": 2 }, { "format": "{int(100,999)}", "weight": 2 },
{ "format": "{int(1000,9999)}", "weight": 0.5 }, { "format": "{int(1000,9999)}", "weight": 0.5 },
{ "format": "{int(10,99)}{int(100,999)}", "weight": 0.3 } { "format": "{int(10,99)}{int(100,999)}", "weight": 0.3 }
], ],
"locality": "{.locality.name}", "locality": "{l.name}",
"region": "{.region.abbr}", "region": "{l..municipality..region.abbr}",
"postal-code": "{.postal-code.code}" "postal-code": "{l.postal-code.code}"
} }
+15
View File
@@ -0,0 +1,15 @@
// Package geous is fejkdata's US places module, whose categories are geo.US.*.
package geous
import (
"embed"
"io/fs"
)
// FS holds the module's categories and its manifest.
//
//go:embed *.json *.tsv .fejkdata.json
var FS embed.FS
// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
func Modules() []fs.FS { return []fs.FS{FS} }
+22
View File
@@ -0,0 +1,22 @@
package geous_test
import (
"io/fs"
"testing"
"github.com/larvit/fejkdata"
geous "github.com/larvit/fejkdata/data/geo/US"
)
func TestFSLoadsByItsManifest(t *testing.T) {
if _, err := fs.Stat(geous.FS, ".fejkdata.json"); err != nil {
t.Fatalf("FS holds no manifest: %v", err)
}
f, err := fejkdata.New(fejkdata.WithDataFS(geous.Modules()...), fejkdata.WithSeed(1))
if err != nil {
t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
}
if len(f.List()) == 0 {
t.Error("List() is empty, want the module's categories")
}
}
+717 -585
View File
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+1113 -1389
View File
File diff suppressed because it is too large Load Diff
+465333 -5902
View File
File diff suppressed because it is too large Load Diff
+25
View File
@@ -0,0 +1,25 @@
{
"index": {
"misc.car": {"paths":["","make","model"]},
"misc.coordinate": {"paths":["","lat","lon"]},
"misc.creditcard": {"paths":["","d"]},
"misc.currency": {"paths":["","code","decimals","name","numeric","symbol"]},
"misc.datetime": {"paths":[""]},
"misc.emoji": {"paths":[""]},
"misc.httpmethod": {"paths":["","idempotent","method","safe"]},
"misc.httpstatus": {"paths":["","code","reason"]},
"misc.language": {"paths":["","code","code3","name"]},
"misc.loglevel": {"paths":["","code","keyword","severity"]},
"misc.mac": {"paths":[""]},
"misc.mimetype": {"paths":["","ext","type"]},
"misc.objectid": {"paths":[""]},
"misc.port": {"paths":["","number","service"]},
"misc.protocol": {"paths":["","keyword","name","number"]},
"misc.territory": {"paths":["","alpha2","alpha3","calling-code","capital","country","currency","flag","languages","name","numeric","timezone","timezone.offset","timezone.territory","timezone.weight","timezone.zone","tld"]},
"misc.timezone": {"parent":"territory","paths":["","offset","territory","weight","zone"]},
"misc.tld": {"paths":["","tld","type","unicode"]},
"misc.useragent": {"paths":["","browser","device","os","ua"]},
"misc.uuid": {"paths":["","variant"]}
},
"namespace": "misc"
}
+1 -1
View File
@@ -71,7 +71,7 @@ JPY 0 Yen 392 ¥
KES 2 Kenyan Shilling 404 KES KES 2 Kenyan Shilling 404 KES
KGS 2 Som 417 ⃀ KGS 2 Som 417 ⃀
KHR 2 Riel 116 ៛ KHR 2 Riel 116 ៛
KMF 0 Comorian Franc 174 CF KMF 0 Comorian Franc 174 CF
KPW 2 North Korean Won 408 ₩ KPW 2 North Korean Won 408 ₩
KRW 0 Won 410 ₩ KRW 0 Won 410 ₩
KWD 3 Kuwaiti Dinar 414 KWD KWD 3 Kuwaiti Dinar 414 KWD
1 code decimals name numeric symbol
71 KES 2 Kenyan Shilling 404 KES
72 KGS 2 Som 417 ⃀
73 KHR 2 Riel 116 ៛
74 KMF 0 Comorian Franc Comorian Franc 174 CF
75 KPW 2 North Korean Won 408 ₩
76 KRW 0 Won 410 ₩
77 KWD 3 Kuwaiti Dinar 414 KWD
+15
View File
@@ -0,0 +1,15 @@
// Package misc is fejkdata's locale-neutral module, whose categories are misc.*.
package misc
import (
"embed"
"io/fs"
)
// FS holds the module's categories and its manifest.
//
//go:embed *.json *.tsv .fejkdata.json
var FS embed.FS
// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
func Modules() []fs.FS { return []fs.FS{FS} }
+22
View File
@@ -0,0 +1,22 @@
package misc_test
import (
"io/fs"
"testing"
"github.com/larvit/fejkdata"
"github.com/larvit/fejkdata/data/misc"
)
func TestFSLoadsByItsManifest(t *testing.T) {
if _, err := fs.Stat(misc.FS, ".fejkdata.json"); err != nil {
t.Fatalf("FS holds no manifest: %v", err)
}
f, err := fejkdata.New(fejkdata.WithDataFS(misc.Modules()...), fejkdata.WithSeed(1))
if err != nil {
t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
}
if len(f.List()) == 0 {
t.Error("List() is empty, want the module's categories")
}
}
+1 -1
View File
@@ -48,7 +48,7 @@ CO COL 57 Bogota CO COP 🇨🇴 es-CO Colombia 170 .co
CR CRI 506 San Jose CR CRC 🇨🇷 es-CR,en Costa Rica 188 .cr CR CRI 506 San Jose CR CRC 🇨🇷 es-CR,en Costa Rica 188 .cr
CU CUB 53 Havana CU CUP 🇨🇺 es-CU,pap Cuba 192 .cu CU CUB 53 Havana CU CUP 🇨🇺 es-CU,pap Cuba 192 .cu
CV CPV 238 Praia CV CVE 🇨🇻 pt-CV Cape Verde 132 .cv CV CPV 238 Praia CV CVE 🇨🇻 pt-CV Cape Verde 132 .cv
CW CUW 599 Willemstad NL XCG 🇨🇼 nl,pap Curaçao 531 .cw CW CUW 599 Willemstad NL XCG 🇨🇼 nl,pap Curaçao 531 .cw
CX CXR 61 Flying Fish Cove AU AUD 🇨🇽 en,zh,ms-CX Christmas Island 162 .cx CX CXR 61 Flying Fish Cove AU AUD 🇨🇽 en,zh,ms-CX Christmas Island 162 .cx
CY CYP 357 Nicosia CY EUR 🇨🇾 el-CY,tr-CY,en Cyprus 196 .cy CY CYP 357 Nicosia CY EUR 🇨🇾 el-CY,tr-CY,en Cyprus 196 .cy
CZ CZE 420 Prague CZ CZK 🇨🇿 cs,sk Czechia 203 .cz CZ CZE 420 Prague CZ CZK 🇨🇿 cs,sk Czechia 203 .cz
1 alpha2 alpha3 calling-code capital country currency flag languages name numeric tld
48 CR CRI 506 San Jose CR CRC 🇨🇷 es-CR,en Costa Rica 188 .cr
49 CU CUB 53 Havana CU CUP 🇨🇺 es-CU,pap Cuba 192 .cu
50 CV CPV 238 Praia CV CVE 🇨🇻 pt-CV Cape Verde 132 .cv
51 CW CUW 599 Willemstad Willemstad NL XCG 🇨🇼 nl,pap Curaçao 531 .cw
52 CX CXR 61 Flying Fish Cove AU AUD 🇨🇽 en,zh,ms-CX Christmas Island 162 .cx
53 CY CYP 357 Nicosia CY EUR 🇨🇾 el-CY,tr-CY,en Cyprus 196 .cy
54 CZ CZE 420 Prague CZ CZK 🇨🇿 cs,sk Czechia 203 .cz
+19
View File
@@ -0,0 +1,19 @@
// Package data lists every module fejkdata ships.
package data
import (
"io/fs"
enus "github.com/larvit/fejkdata/data/en_US"
geose "github.com/larvit/fejkdata/data/geo/SE"
geous "github.com/larvit/fejkdata/data/geo/US"
"github.com/larvit/fejkdata/data/misc"
svse "github.com/larvit/fejkdata/data/sv_SE"
)
//go:generate env REGENERATE=1 go test -run ^TestShippedManifestsAreCurrent$ ..
// Modules returns every module fejkdata ships, to pass to fejkdata.WithDataFS.
func Modules() []fs.FS {
return []fs.FS{enus.FS, geose.FS, geous.FS, misc.FS, svse.FS}
}
+27
View File
@@ -0,0 +1,27 @@
{
"index": {
"sv_SE.address": {"paths":["","locality","postal-code","street","street-number"]},
"sv_SE.birth-number": {"parent":"sex","paths":["","number","sex"]},
"sv_SE.color": {"paths":[""]},
"sv_SE.company": {"paths":["","base","suffix"]},
"sv_SE.date": {"paths":[""]},
"sv_SE.email": {"paths":["","domain","local","local.n"]},
"sv_SE.first-name": {"parent":"sex","paths":["","count","name","sex"]},
"sv_SE.ip": {"paths":[""]},
"sv_SE.last-name": {"paths":["","count","name"]},
"sv_SE.person": {"paths":["","first","last","prefix","sex"]},
"sv_SE.personnummer": {"paths":[""]},
"sv_SE.phone": {"paths":["","a","b","c","prefix"]},
"sv_SE.price": {"paths":["","amt","ore"]},
"sv_SE.samordningsnummer": {"paths":[""]},
"sv_SE.sentence": {"paths":["","adj","noun","prep","verb"]},
"sv_SE.sex": {"paths":["","birth-number","birth-number.number","birth-number.sex","code","first-name","first-name.count","first-name.name","first-name.sex","name"]},
"sv_SE.time": {"paths":[""]},
"sv_SE.title": {"paths":["","name","share"]},
"sv_SE.url": {"paths":["","host","path"]},
"sv_SE.username": {"paths":["","n"]},
"sv_SE.version": {"paths":["","n","pre","suffix"]},
"sv_SE.word": {"paths":[""]}
},
"namespace": "sv_SE"
}
+5 -5
View File
@@ -1,7 +1,7 @@
{ {
"format": "{street} {street-number}\n{postal-code} {locality}", "format": "{/geo.SE.address as a}{a}",
"street": "{/geo.SE.address.street}", "street": "{a.street}",
"street-number": "{/geo.SE.address.street-number}", "street-number": "{a.street-number}",
"postal-code": "{/geo.SE.address.postal-code}", "postal-code": "{a.postal-code}",
"locality": "{/geo.SE.address.locality}" "locality": "{a.locality}"
} }
+19
View File
@@ -0,0 +1,19 @@
// Package svse is fejkdata's sv_SE locale module, whose categories are sv_SE.*; its address reads
// geo.SE, which package geose holds.
package svse
import (
"embed"
"io/fs"
geose "github.com/larvit/fejkdata/data/geo/SE"
)
// FS holds the locale alone, its categories and its manifest; pass Modules unless your own
// data stands in for geo.SE.
//
//go:embed *.json *.tsv .fejkdata.json
var FS embed.FS
// Modules returns FS and every module it reads, to pass to fejkdata.WithDataFS.
func Modules() []fs.FS { return append([]fs.FS{FS}, geose.Modules()...) }
+22
View File
@@ -0,0 +1,22 @@
package svse_test
import (
"io/fs"
"testing"
"github.com/larvit/fejkdata"
svse "github.com/larvit/fejkdata/data/sv_SE"
)
func TestFSLoadsByItsManifest(t *testing.T) {
if _, err := fs.Stat(svse.FS, ".fejkdata.json"); err != nil {
t.Fatalf("FS holds no manifest: %v", err)
}
f, err := fejkdata.New(fejkdata.WithDataFS(svse.Modules()...), fejkdata.WithSeed(1))
if err != nil {
t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
}
if len(f.List()) == 0 {
t.Error("List() is empty, want the module's categories")
}
}
+2 -2
View File
@@ -1,7 +1,7 @@
{ {
"format": "{prefix}{first} {last}", "format": "{prefix}{first} {last}",
"first": "{.first-name.name}", "first": "{.first-name as n}{n.name}",
"last": "{.last-name.name}", "last": "{.last-name.name}",
"prefix": [null, { "format": "{.title.name} ", "weight": 0.05 }], "prefix": [null, { "format": "{.title.name} ", "weight": 0.05 }],
"sex": "{.sex.name}" "sex": "{n..sex.name}"
} }
+243 -38
View File
@@ -1,7 +1,10 @@
package fejkdata package fejkdata
import ( import (
"io/fs"
"os"
"regexp" "regexp"
"slices"
"strconv" "strconv"
"strings" "strings"
"testing" "testing"
@@ -48,7 +51,7 @@ func TestShippedDataCategories(t *testing.T) {
regexp.MustCompile(`^\d{1,3}( \d{3})?(,\d{2})? kr$`)}, regexp.MustCompile(`^\d{1,3}( \d{3})?(,\d{2})? kr$`)},
} }
f := newGenerator(t, "data", WithSeed(1)) f := shipped(t, WithSeed(1))
for _, c := range cases { for _, c := range cases {
for i := 0; i < 200; i++ { for i := 0; i < 200; i++ {
if v := fake(t, f, "en_US."+c.path); !c.en.MatchString(v) { if v := fake(t, f, "en_US."+c.path); !c.en.MatchString(v) {
@@ -61,11 +64,11 @@ func TestShippedDataCategories(t *testing.T) {
} }
} }
// TestShippedMiscCategories covers the locale-neutral data/misc folder: a proper // TestShippedMiscCategories covers the locale-neutral misc module: a proper
// v4 UUID (version/variant nibbles fixed), a MAC address, and credit-card numbers // v4 UUID (version/variant nibbles fixed), a MAC address, and credit-card numbers
// whose trailing {luhn()} check passes. // whose trailing {luhn()} check passes.
func TestShippedMiscCategories(t *testing.T) { func TestShippedMiscCategories(t *testing.T) {
f := newGenerator(t, "data/misc", WithSeed(1)) f := newBareModule(t, "misc", WithSeed(1))
v4 := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`) v4 := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
mac := regexp.MustCompile(`^([0-9a-f]{2}:){5}[0-9a-f]{2}$`) mac := regexp.MustCompile(`^([0-9a-f]{2}:){5}[0-9a-f]{2}$`)
objectid := regexp.MustCompile(`^[0-9a-f]{24}$`) objectid := regexp.MustCompile(`^[0-9a-f]{24}$`)
@@ -93,7 +96,7 @@ func TestShippedMiscCategories(t *testing.T) {
// codes follow their standard shape, dotted sub-paths resolve, emoji is a real // codes follow their standard shape, dotted sub-paths resolve, emoji is a real
// glyph, and a coordinate parses to a point within geographic bounds. // glyph, and a coordinate parses to a point within geographic bounds.
func TestShippedMiscReferenceData(t *testing.T) { func TestShippedMiscReferenceData(t *testing.T) {
f := newGenerator(t, "data/misc", WithSeed(1)) f := newBareModule(t, "misc", WithSeed(1))
re := map[string]*regexp.Regexp{ re := map[string]*regexp.Regexp{
"currency": regexp.MustCompile(`^[A-Z]{3}$`), "currency": regexp.MustCompile(`^[A-Z]{3}$`),
"currency.name": regexp.MustCompile(`\p{L}`), "currency.name": regexp.MustCompile(`\p{L}`),
@@ -156,7 +159,7 @@ func TestShippedMiscReferenceData(t *testing.T) {
// TestSwedishPersonNamesHaveNoTripleLetter pins an orthographic rule the shape // TestSwedishPersonNamesHaveNoTripleLetter pins an orthographic rule the shape
// regexes miss: no generated name repeats a character three times over. // regexes miss: no generated name repeats a character three times over.
func TestSwedishPersonNamesHaveNoTripleLetter(t *testing.T) { func TestSwedishPersonNamesHaveNoTripleLetter(t *testing.T) {
f := newGenerator(t, "data", WithSeed(11)) f := shipped(t, WithSeed(11))
for _, path := range []string{"sv_SE.person", "sv_SE.person.last"} { for _, path := range []string{"sv_SE.person", "sv_SE.person.last"} {
for i := 0; i < 20000; i++ { for i := 0; i < 20000; i++ {
name := fake(t, f, path) name := fake(t, f, path)
@@ -175,7 +178,7 @@ func TestSwedishPersonNamesHaveNoTripleLetter(t *testing.T) {
// male), the trailing digit is a Luhn checksum over the other nine, and a // male), the trailing digit is a Luhn checksum over the other nine, and a
// samordningsnummer is the same with 60 added to the day. // samordningsnummer is the same with 60 added to the day.
func TestSwedishPersonnummer(t *testing.T) { func TestSwedishPersonnummer(t *testing.T) {
sv := newGenerator(t, "data", WithSeed(1)) sv := shipped(t, WithSeed(1))
re := regexp.MustCompile(`^\d{6}-23[89]\d$`) re := regexp.MustCompile(`^\d{6}-23[89]\d$`)
sawLongMonthEnd, birth := false, map[string]bool{} sawLongMonthEnd, birth := false, map[string]bool{}
for i := 0; i < 2000; i++ { for i := 0; i < 2000; i++ {
@@ -208,20 +211,13 @@ func TestSwedishPersonnummer(t *testing.T) {
if !birth["238"] || !birth["239"] { if !birth["238"] || !birth["239"] {
t.Fatalf("birth numbers drawn %v, want both 238 and 239", birth) t.Fatalf("birth numbers drawn %v, want both 238 and 239", birth)
} }
for i := 0; i < 200; i++ {
got := fakeTemplate(t, sv, `{/sv_SE.person.sex} {/sv_SE.personnummer}`)
sex, id, _ := strings.Cut(got, " ")
if want := map[string]string{"kvinna": "238", "man": "239"}[sex]; want == "" || !strings.Contains(id, "-"+want) {
t.Fatalf("%q: a person and a personnummer in one render disagree on sex", got)
}
}
} }
// TestShippedUSTaxIds pins the SSA and IRS ranges: an SSN's area is 001-899 but // TestShippedUSTaxIds pins the SSA and IRS ranges: an SSN's area is 001-899 but
// 666, its group 01-99 and its serial 0001-9999; an ITIN is 9XX-GG-XXXX with GG // 666, its group 01-99 and its serial 0001-9999; an ITIN is 9XX-GG-XXXX with GG
// in 50-65, 70-88, 90-92 or 94-99. // in 50-65, 70-88, 90-92 or 94-99.
func TestShippedUSTaxIds(t *testing.T) { func TestShippedUSTaxIds(t *testing.T) {
f := newGenerator(t, "data", WithSeed(5)) f := shipped(t, WithSeed(5))
ssn := regexp.MustCompile(`^(\d{3})-(\d{2})-(\d{4})$`) ssn := regexp.MustCompile(`^(\d{3})-(\d{2})-(\d{4})$`)
itin := regexp.MustCompile(`^9\d{2}-(5\d|6[0-5]|7\d|8[0-8]|9[0-24-9])-\d{4}$`) itin := regexp.MustCompile(`^9\d{2}-(5\d|6[0-5]|7\d|8[0-8]|9[0-24-9])-\d{4}$`)
for i := 0; i < 2000; i++ { for i := 0; i < 2000; i++ {
@@ -245,7 +241,7 @@ func TestShippedUSTaxIds(t *testing.T) {
// TestShippedPersonNames pins the name tables: a sex pins its first names, a name // TestShippedPersonNames pins the name tables: a sex pins its first names, a name
// both sexes carry appears under both, and the record's sex agrees with its name. // both sexes carry appears under both, and the record's sex agrees with its name.
func TestShippedPersonNames(t *testing.T) { func TestShippedPersonNames(t *testing.T) {
f := newGenerator(t, "data", WithSeed(9)) f := shipped(t, WithSeed(9))
for path, want := range map[string]string{ for path, want := range map[string]string{
"sv_SE.sex[f]": "kvinna", "sv_SE.sex[f]": "kvinna",
"sv_SE.sex[man].code": "m", "sv_SE.sex[man].code": "m",
@@ -273,24 +269,41 @@ func TestShippedPersonNames(t *testing.T) {
seen := map[string]bool{} seen := map[string]bool{}
for i := 0; i < 2000; i++ { for i := 0; i < 2000; i++ {
seen[fake(t, f, locale+".sex[f].first-name")] = true seen[fake(t, f, locale+".sex[f].first-name")] = true
first, sex := fake(t, f, locale+".person.first"), fake(t, f, locale+".person.sex") got := fakeTemplate(t, f, `{/`+locale+`.person as p}{p.first}|{p.sex}`)
if first == "" || sex == "" { first, sex, _ := strings.Cut(got, "|")
t.Fatalf("%s.person lacks a first name or a sex", locale) fake(t, f, locale+".sex["+sex+"].first-name["+first+"]")
}
} }
if len(seen) < 300 || !seen["Anna"] && locale == "sv_SE" || !seen["Mary"] && locale == "en_US" { if len(seen) < 300 || !seen["Anna"] && locale == "sv_SE" || !seen["Mary"] && locale == "en_US" {
t.Fatalf("%s female first names: %d distinct in 2000, want a weighted register", locale, len(seen)) t.Fatalf("%s female first names: %d distinct in 2000, want a weighted register", locale, len(seen))
} }
got := fakeTemplate(t, f, `{/`+locale+`.sex.code}|{/`+locale+`.person.first}`) }
code, first, _ := strings.Cut(got, "|") }
if v := fake(t, f, locale+".sex["+code+"].first-name["+first+"]"); v != first {
t.Fatalf("%s: %q drawn as a %s name is not one", locale, first, code) // TestShippedPersonIsOneNamedPick pins that a person's first name, sex and title come
// from one pick, and that two names bound to one person category are two people.
func TestShippedPersonIsOneNamedPick(t *testing.T) {
f := shipped(t, WithSeed(9))
for _, locale := range []string{"sv_SE", "en_US"} {
apart := false
for i := 0; i < 300; i++ {
got := fakeTemplate(t, f, `{/`+locale+`.person as a}{/`+locale+`.person as b}{a.first}|{a.sex}|{a.prefix}|{b.first}|{b.sex}`)
parts := strings.Split(got, "|")
for _, p := range [][2]string{{parts[0], parts[1]}, {parts[3], parts[4]}} {
fake(t, f, locale+".sex["+p[1]+"].first-name["+p[0]+"]")
}
if title := strings.TrimSpace(parts[2]); title != "" && locale == "en_US" {
fake(t, f, "en_US.sex["+parts[1]+"].title["+title+"]")
}
apart = apart || parts[0] != parts[3]
}
if !apart {
t.Errorf("%s: two names bound to person drew one first name in 300 renders, want two people", locale)
} }
} }
} }
func TestShippedSwedishPhone(t *testing.T) { func TestShippedSwedishPhone(t *testing.T) {
f := newGenerator(t, "data", WithSeed(11)) f := shipped(t, WithSeed(11))
re := regexp.MustCompile(`^0\d{1,2}-\d{3} \d{2} \d{2}$`) re := regexp.MustCompile(`^0\d{1,2}-\d{3} \d{2} \d{2}$`)
for i := 0; i < 50; i++ { for i := 0; i < 50; i++ {
if n := fake(t, f, "sv_SE.phone"); !re.MatchString(n) { if n := fake(t, f, "sv_SE.phone"); !re.MatchString(n) {
@@ -300,7 +313,7 @@ func TestShippedSwedishPhone(t *testing.T) {
} }
func TestShippedSwedishAddress(t *testing.T) { func TestShippedSwedishAddress(t *testing.T) {
f := newGenerator(t, "data", WithSeed(3)) f := shipped(t, WithSeed(3))
digit := regexp.MustCompile(`\d`) digit := regexp.MustCompile(`\d`)
for i := 0; i < 30; i++ { for i := 0; i < 30; i++ {
a := fake(t, f, "sv_SE.address") a := fake(t, f, "sv_SE.address")
@@ -317,8 +330,170 @@ func TestShippedSwedishAddress(t *testing.T) {
} }
} }
// TestShippedAddressIsOneNamedPick pins that an address's street, postal code,
// locality and region are one place in the geo tables, read through a name in a template or
// across a struct's tags, and that a reference beside the name is another address.
func TestShippedAddressIsOneNamedPick(t *testing.T) {
f := shipped(t, WithSeed(3))
for _, c := range []struct{ locale, country, template, largest string }{
{"sv_SE", "SE", "{a.street}|{a.postal-code}|{a.locality}|", "Stockholm"},
{"en_US", "US", "{a.street}|{a.postal-code}|{a.locality}|{a.region}", "New York"},
} {
places := geoPlaces(t, c.country)
apart, localities := false, map[string]int{}
for i := 0; i < 2000; i++ {
got := fakeTemplate(t, f, `{/`+c.locale+`.address as a}`+c.template+`|{/`+c.locale+`.address.street}`)
parts := strings.Split(got, "|")
if !places[strings.Join(parts[:4], "|")] {
t.Fatalf("%s.address: %q is no street, postal code, locality and region of one place", c.locale, got)
}
apart = apart || parts[0] != parts[4]
localities[parts[2]]++
}
if !apart {
t.Errorf("%s: a reference beside a name drew the name's street in 2000 renders, want two addresses", c.locale)
}
if c.country == "SE" {
var row struct {
Street string `fake:"{/sv_SE.address as a}{a.street}"`
PostalCode string `fake:"{a.postal-code}"`
Locality string `fake:"{a.locality}"`
}
for i := 0; i < 200; i++ {
if err := f.FakeStruct(&row); err != nil {
t.Fatal(err)
}
if got := row.Street + "|" + row.PostalCode + "|" + row.Locality + "|"; !places[got] {
t.Fatalf("sv_SE.address: struct tags reading one name, %q, are no street, postal code and locality of one place", got)
}
}
}
for l, n := range localities {
if n > localities[c.largest] {
t.Errorf("%s.address drew %s %d times and %s %d, want the locality weighted by population", c.locale, l, n, c.largest, localities[c.largest])
}
}
}
}
// 0644000 is Los Angeles.
func TestShippedPlacesSpreadAddresses(t *testing.T) {
for country, city := range map[string]struct {
key string
least int
}{"SE": {"Göteborg", 1500}, "US": {"0644000", 2500}} {
streets := 0
for _, r := range geoRows(t, country, "street") {
if r[1] == city.key {
streets++
}
}
if streets < city.least {
t.Errorf("geo.%s: %s ships %d streets, want at least %d", country, city.key, streets, city.least)
}
}
header, _, _ := strings.Cut(geoTSV(t, "US", "locality"), "\n")
population := slices.Index(strings.Split(header, "\t"), "population")
if population < 0 {
t.Fatal("geo.US locality.tsv has no population column")
}
for _, r := range geoRows(t, "US", "locality") {
n, err := strconv.Atoi(r[population])
if err != nil {
t.Fatal(err)
}
if n < 2500 {
return
}
}
t.Error("geo.US: every locality holds 2500 people or more, want places drawn down to villages")
}
// TestShippedCapitalsAndBigPlaces pins state capitals and places of 100,000 or more that a
// draw by population leaves out.
func TestShippedCapitalsAndBigPlaces(t *testing.T) {
state := map[string]string{}
for _, r := range geoRows(t, "US", "municipality") {
state[r[0]] = r[2]
}
shipped := map[string]bool{}
for _, r := range geoRows(t, "US", "locality") {
shipped[r[1]+", "+state[r[2]]] = true
}
for _, place := range []string{
"Albany, NY", "Augusta, ME", "Baton Rouge, LA", "Bismarck, ND", "Boise, ID", "Charleston, WV",
"Frankfort, KY", "Harrisburg, PA", "Helena, MT", "Juneau, AK", "Kansas City, KS", "Lafayette, LA",
"Lexington, KY", "Montpelier, VT", "New Orleans, LA", "Pierre, SD", "St. Paul, MN", "Stockton, CA",
"Ventura, CA",
} {
if !shipped[place] {
t.Errorf("geo.US: no locality %s, want every state capital and place of 100,000 or more", place)
}
}
}
func TestShippedSwedishStreetsAreStreetNames(t *testing.T) {
refused := map[string]bool{
"Färjeled Henriksdalsbryggan - Barnängsbryggan": true, "HÄLLSÄTTERSLEDEN": true, "Kärrs VSF": true,
"Länsväg 694": true, "NN": true, "Risövägen,Dejstolvägen,Runvägen": true, "tornslingan": true,
"Öresundsvägen, Bondevägen": true,
}
for _, r := range geoRows(t, "SE", "street") {
if refused[r[0]] {
t.Errorf("geo.SE: street %q in %s, want no road number, ferry route, label, placeholder, two names or miscased name", r[0], r[1])
}
}
}
func geoTSV(t *testing.T, country, table string) string {
b, err := os.ReadFile("data/geo/" + country + "/" + table + ".tsv")
if err != nil {
t.Fatal(err)
}
return strings.TrimSpace(string(b))
}
func geoRows(t *testing.T, country, table string) [][]string {
var out [][]string
for _, line := range strings.Split(geoTSV(t, country, table), "\n")[1:] {
out = append(out, strings.Split(line, "\t"))
}
return out
}
// geoPlaces reads a country's geo tables into every "street|postal-code|locality|region"
// one locality holds, the region empty for SE, whose address carries none.
func geoPlaces(t *testing.T, country string) map[string]bool {
rows := func(table string) [][]string { return geoRows(t, country, table) }
name, region := map[string]string{}, map[string]string{}
if country == "US" {
county := map[string]string{}
for _, r := range rows("municipality") {
county[r[0]] = r[2]
}
for _, r := range rows("locality") {
name[r[0]], region[r[0]] = r[1], county[r[2]]
}
} else {
for _, r := range rows("locality") {
name[r[0]] = r[0]
}
}
codes := map[string][]string{}
for _, r := range rows("postal-code") {
codes[r[1]] = append(codes[r[1]], r[0])
}
places := map[string]bool{}
for _, r := range rows("street") {
for _, code := range codes[r[1]] {
places[strings.Join([]string{r[0], code, name[r[1]], region[r[1]]}, "|")] = true
}
}
return places
}
func TestShippedPersonHasParts(t *testing.T) { func TestShippedPersonHasParts(t *testing.T) {
f := newGenerator(t, "data", WithSeed(7)) f := shipped(t, WithSeed(7))
for _, path := range []string{"sv_SE.person", "en_US.person"} { for _, path := range []string{"sv_SE.person", "en_US.person"} {
for i := 0; i < 30; i++ { for i := 0; i < 30; i++ {
if name := fake(t, f, path); len(name) < 3 || !regexp.MustCompile(`\S \S`).MatchString(name) { if name := fake(t, f, path); len(name) < 3 || !regexp.MustCompile(`\S \S`).MatchString(name) {
@@ -328,20 +503,50 @@ func TestShippedPersonHasParts(t *testing.T) {
} }
} }
// NANPA's central office code records mark N11, 555, 950, 958, 959 and 976 unassignable.
func TestShippedUSPhone(t *testing.T) { func TestShippedUSPhone(t *testing.T) {
f := newGenerator(t, "data", WithSeed(11)) f := shipped(t, WithSeed(11))
re := regexp.MustCompile(`^(\(\d{3}\) \d{3}-\d{4}|\d{3}-\d{3}-\d{4})$`) re := regexp.MustCompile(`^(?:\(\d{3}\) |\d{3}-)([2-9]\d{2})-\d{4}$`)
for i := 0; i < 50; i++ { unassignable := map[string]bool{"555": true, "950": true, "958": true, "959": true, "976": true}
if n := fake(t, f, "en_US.phone"); !re.MatchString(n) { for i := 0; i < 2000; i++ {
n := fake(t, f, "en_US.phone")
m := re.FindStringSubmatch(n)
if m == nil {
t.Fatalf("phone %q does not match %s", n, re) t.Fatalf("phone %q does not match %s", n, re)
} }
if exchange := m[1]; exchange[1:] == "11" || unassignable[exchange] {
t.Fatalf("phone %q has the unassignable exchange %s", n, exchange)
}
} }
} }
// TestShippedNamespacedTree loads the whole data/ tree (not a single locale) and func TestShippedTableCellsAreTrimmed(t *testing.T) {
// reaches each locale through its folder segment: data/sv_SE/person -> sv_SE.person. err := fs.WalkDir(os.DirFS("data"), ".", func(path string, d fs.DirEntry, err error) error {
if err != nil || !strings.HasSuffix(path, ".tsv") {
return err
}
b, err := fs.ReadFile(os.DirFS("data"), path)
if err != nil {
return err
}
for i, line := range strings.Split(string(b), "\n") {
for _, cell := range strings.Split(line, "\t") {
if cell != strings.TrimSpace(cell) {
t.Errorf("data/%s:%d: cell %q is padded", path, i+1, cell)
}
}
}
return nil
})
if err != nil {
t.Fatal(err)
}
}
// TestShippedNamespacedTree loads every shipped module and reaches each locale through its
// namespace: data/sv_SE/person.json, mounted under sv_SE, is sv_SE.person.
func TestShippedNamespacedTree(t *testing.T) { func TestShippedNamespacedTree(t *testing.T) {
f := newGenerator(t, "data", WithSeed(1)) f := shipped(t, WithSeed(1))
for _, path := range []string{"sv_SE.person", "en_US.person", "sv_SE.address.locality"} { for _, path := range []string{"sv_SE.person", "en_US.person", "sv_SE.address.locality"} {
if got := fake(t, f, path); got == "" { if got := fake(t, f, path); got == "" {
t.Fatalf("Fake(%q) returned empty", path) t.Fatalf("Fake(%q) returned empty", path)
@@ -385,7 +590,7 @@ func luhnValid(s string) bool {
var swedishName = regexp.MustCompile(`^\p{L}+([ -]\p{L}+)*$`) var swedishName = regexp.MustCompile(`^\p{L}+([ -]\p{L}+)*$`)
func TestShippedStreetIsARegisteredName(t *testing.T) { func TestShippedStreetIsARegisteredName(t *testing.T) {
f := newGenerator(t, "data", WithSeed(5)) f := shipped(t, WithSeed(5))
street := regexp.MustCompile(`^\p{L}[\p{L}\d:.-]*([ -][\p{L}\d:.-]+)*$`) street := regexp.MustCompile(`^\p{L}[\p{L}\d:.-]*([ -][\p{L}\d:.-]+)*$`)
for i := 0; i < 300; i++ { for i := 0; i < 300; i++ {
if s := fake(t, f, "sv_SE.address.street"); !street.MatchString(s) { if s := fake(t, f, "sv_SE.address.street"); !street.MatchString(s) {
@@ -397,7 +602,7 @@ func TestShippedStreetIsARegisteredName(t *testing.T) {
func TestShippedLastNameComposition(t *testing.T) { func TestShippedLastNameComposition(t *testing.T) {
// last is a choice of patronymic {first}sson templates, compound // last is a choice of patronymic {first}sson templates, compound
// {first}{last} templates and literal surnames. // {first}{last} templates and literal surnames.
f := newGenerator(t, "data", WithSeed(6)) f := shipped(t, WithSeed(6))
for i := 0; i < 300; i++ { for i := 0; i < 300; i++ {
if s := fake(t, f, "sv_SE.person.last"); !swedishName.MatchString(s) { if s := fake(t, f, "sv_SE.person.last"); !swedishName.MatchString(s) {
t.Fatalf("last name %q is not a Swedish surname", s) t.Fatalf("last name %q is not a Swedish surname", s)
@@ -407,7 +612,7 @@ func TestShippedLastNameComposition(t *testing.T) {
func TestShippedStreetNumberFormats(t *testing.T) { func TestShippedStreetNumberFormats(t *testing.T) {
// Reachable via a hyphenated path; covers all five weighted number variants. // Reachable via a hyphenated path; covers all five weighted number variants.
f := newGenerator(t, "data", WithSeed(8)) f := shipped(t, WithSeed(8))
re := regexp.MustCompile(`^[1-9]\d{0,2}[A-Z]?$`) re := regexp.MustCompile(`^[1-9]\d{0,2}[A-Z]?$`)
for i := 0; i < 300; i++ { for i := 0; i < 300; i++ {
if n := fake(t, f, "sv_SE.address.street-number"); !re.MatchString(n) { if n := fake(t, f, "sv_SE.address.street-number"); !re.MatchString(n) {
@@ -419,11 +624,11 @@ func TestShippedStreetNumberFormats(t *testing.T) {
// TestShippedUSTitleAgreesWithSex pins that one person's title and sex agree, and // TestShippedUSTitleAgreesWithSex pins that one person's title and sex agree, and
// that the everyday titles are reachable. // that the everyday titles are reachable.
func TestShippedUSTitleAgreesWithSex(t *testing.T) { func TestShippedUSTitleAgreesWithSex(t *testing.T) {
f := newGenerator(t, "data", WithSeed(4)) f := shipped(t, WithSeed(4))
female := map[string]bool{"Miss": true, "Mrs": true, "Ms": true} female := map[string]bool{"Miss": true, "Mrs": true, "Ms": true}
seen := map[string]bool{} seen := map[string]bool{}
for i := 0; i < 3000; i++ { for i := 0; i < 3000; i++ {
got := fakeTemplate(t, f, `{/en_US.person.prefix}|{/en_US.person.sex}`) got := fakeTemplate(t, f, `{/en_US.person as p}{p.prefix}|{p.sex}`)
title, sex, _ := strings.Cut(got, "|") title, sex, _ := strings.Cut(got, "|")
title = strings.TrimSpace(title) title = strings.TrimSpace(title)
seen[title] = true seen[title] = true
+161 -113
View File
@@ -3,97 +3,83 @@ package fejkdata
import ( import (
"errors" "errors"
"fmt" "fmt"
"github.com/larvit/fejkdata/internal/datatype"
"github.com/larvit/fejkdata/internal/grammar"
"github.com/larvit/fejkdata/internal/invariant"
"github.com/larvit/fejkdata/internal/jsonvalue"
) )
// DataType is what a record column holds, which decides how a record writes its value. // DataType is what a record column holds, which decides how a record writes its value. It
type DataType int // prints as data spells it: string, integer, number or boolean.
type DataType = datatype.DataType
// The datatypes a column declares with "datatype"; a column without one is a string. // The datatypes a column declares with "datatype"; a column without one is a string.
const ( const (
DataTypeString DataType = iota DataTypeString DataType = datatype.String
DataTypeInteger DataTypeInteger DataType = datatype.Integer
DataTypeNumber DataTypeNumber DataType = datatype.Number
DataTypeBoolean DataTypeBoolean DataType = datatype.Boolean
) )
var ( // datatypeOf reads a template's "datatype", nil where it carries none or sits outside a column.
dataTypeNames = [...]string{"string", "integer", "number", "boolean"} func datatypeOf(m map[string]any, pos position) (*DataType, error) {
dataTypeNouns = [...]string{"text", "an integer", "a number", "a boolean"}
)
// String is the datatype as data spells it.
func (d DataType) String() string {
if d < 0 || int(d) >= len(dataTypeNames) {
return fmt.Sprintf("DataType(%d)", int(d))
}
return dataTypeNames[d]
}
// position is where a JSON value sits, which decides whether it may carry a datatype or
// be null.
type position int
const (
inFormat position = iota // rendered by a format, so neither
atTop // a category or an inline template, whose fields may be columns
inColumn // a column, or a choice item standing in for one
)
// datatypeOf reads a template's "datatype" (default DataTypeString).
func datatypeOf(m map[string]any, pos position) (DataType, error) {
v, ok := m["datatype"] v, ok := m["datatype"]
if !ok { if !ok {
return DataTypeString, nil return nil, nil
} }
name, ok := v.(string) name, ok := v.(string)
if !ok { if !ok {
return 0, fmt.Errorf("datatype must be a string, got %T", v) return nil, fmt.Errorf("datatype must be a string, not %s", jsonvalue.Kind(v))
} }
if name == DataTypeString.String() { for d := DataTypeString; d < datatype.Count; d++ {
return 0, fmt.Errorf("datatype %q is the default, so it has no effect; drop it", name)
}
for d := DataTypeInteger; d <= DataTypeBoolean; d++ {
if name != d.String() { if name != d.String() {
continue continue
} }
if pos != inColumn { if !pos.column() {
return 0, errors.New("datatype only types a record column — a field of the top-level template — so it has no effect here") return nil, nil
} }
return d, nil return &d, nil
} }
return 0, fmt.Errorf(`datatype takes "integer", "number" or "boolean", got %q`, name) return nil, fmt.Errorf(`datatype takes "string", "integer", "number" or "boolean", got %q`, name)
} }
// checkColumns rejects a record column whose items hold different datatypes, checking a column // checkColumns rejects a record column whose items hold different datatypes, checking a column
// after the columns its items read, so a column read is named before its readers. // after the columns its items read, so a column read is named before its readers.
func checkColumns(s nodeScope) error { func checkColumns(s nodeSet) error {
checked := map[node]bool{} checked := map[node]bool{}
var check func(path, name string, column node) error var check func(label, name string, column node) error
check = func(path, name string, column node) error { check = func(label, name string, column node) error {
if checked[column] { if checked[column] {
return nil return nil
} }
checked[column] = true checked[column] = true
items, _ := columnItems(column) items, _ := columnItems(column)
for _, it := range items { for _, it := range items {
if r := it.readsColumn; r != nil { if r := it.readsColumn; r != nil && r.category != "" {
if err := check(r.a.key[1:], r.a.tail[0], r.column); err != nil { if err := check(r.category, r.field, r.column); err != nil {
return err return err
} }
} }
} }
if _, err := columnDatatype(column); err != nil { if len(items) == 0 {
return fmt.Errorf("%s: field %q: %w", path, name, err) return nil
}
first := columnDatatype(column)
for _, it := range items[1:] {
if d := itemDatatype(it); d != first {
return fmt.Errorf("%s: field %q: %w", label, name, disagreement(items[0], first, it, d))
}
} }
return nil return nil
} }
return s(func(path string, n node) error { return s(func(label string, n node) error {
t, ok := n.(*template) t, ok := n.(*template)
if !ok || !t.record { if !ok || !t.isRecord {
return nil return nil
} }
for _, name := range recordColumns(t) { for _, name := range sortedNames(t.fields) {
if err := check(path, name, t.fields[name]); err != nil { if err := check(label, name, t.fields[name]); err != nil {
return err return err
} }
} }
@@ -101,41 +87,25 @@ func checkColumns(s nodeScope) error {
}) })
} }
// columnDatatype is the datatype a column's items hold. They must agree, since a // columnDatatype is the datatype a column holds, which its first item decides: checkColumns refuses
// column holds one; a column only ever null is a string. // a column whose items disagree. A table's column, like one only ever null, has no item and is a string.
func columnDatatype(n node) (DataType, error) { func columnDatatype(n node) DataType {
items, _ := columnItems(n) items, _ := columnItems(n)
if len(items) == 0 { if len(items) == 0 {
return DataTypeString, nil return DataTypeString
} }
first := itemDatatype(items[0]) return itemDatatype(items[0])
for _, t := range items[1:] {
if d := itemDatatype(t); d != first {
return first, disagreement(items[0], first, t, d)
}
}
return first, nil
} }
// itemDatatype is the datatype a column item declares, else that of the column it is. // itemDatatype is the datatype a column item declares, else that of the column it reads.
func itemDatatype(t *template) DataType { func itemDatatype(t *template) DataType {
if t.datatype != DataTypeString { switch {
return t.datatype case t.datatype != nil:
return *t.datatype
case t.readsColumn != nil:
return columnDatatype(t.readsColumn.column)
} }
return readDatatype(t) return DataTypeString
}
// readDatatype is the datatype of the column t is, reading it by one reference alone; a string
// when t reads none.
func readDatatype(t *template) DataType {
if t.readsColumn == nil {
return DataTypeString
}
items, _ := columnItems(t.readsColumn.column)
if len(items) == 0 {
return DataTypeString
}
return itemDatatype(items[0]) // checkColumns refuses that column where its items disagree
} }
// columnItems is a column's template items, its choices unwrapped, and whether one is null. // columnItems is a column's template items, its choices unwrapped, and whether one is null.
@@ -149,45 +119,91 @@ func columnItems(n node) (items []*template, nullable bool) {
} }
case *template: case *template:
items = append(items, n) items = append(items, n)
case *null: case *nullItem:
nullable = true nullable = true
case *tableColumn:
default:
panic(invariant.Broken("columnItems has no case for node %T", n))
} }
} }
collect(n) collect(n)
return items, nullable return items, nullable
} }
// disagreement names the fix for two items of one column holding different datatypes. // itemKind is how a column item comes by its datatype, ranked in this order: none, read from a typed column, or declared.
func disagreement(a *template, da DataType, b *template, db DataType) error { type itemKind int
bare, typed, want := b, a, da
if da == DataTypeString { const (
bare, typed, want = a, b, db kindText itemKind = iota
} kindReads
kindDeclares
)
func kindOf(t *template) itemKind {
switch { switch {
case da != DataTypeString && db != DataTypeString: case t.datatype != nil:
return bothTyped(a, da, b, db) return kindDeclares
case typed.datatype == DataTypeString && (&valueProof{}).columnItem(bare).not[want] != "": case itemDatatype(t) != DataTypeString:
return fmt.Errorf(`item %q is not %s, the datatype item %q takes from the column it reads; to read that column as text, %s`, bare.format, dataTypeNouns[want], typed.format, asText(typed)) return kindReads
case bare.fromString: // an object may carry a weight, which this spelling would drop
return fmt.Errorf(`item %q declares no datatype, and a column holds one; write it as {"format":%q,"datatype":%q}`, bare.format, bare.format, want)
} }
return fmt.Errorf(`item %q declares no datatype beside one holding %s; a column holds one, so give it "datatype": %q`, bare.format, want, want) return kindText
} }
// bothTyped names the fix for two items holding different datatypes: the one taking its datatype // disagreement names the fix for items a and b of one column holding different datatypes, fixing
// from the column it reads is typed as the other where its values prove it, else read as text. // the one that declares least, a when both declare alike.
func bothTyped(a *template, da DataType, b *template, db DataType) error { func disagreement(a *template, da DataType, b *template, db DataType) error {
read, other := a, db c := clash{fix: a, other: b, want: db, held: fmt.Sprintf("its items hold %s and %s; a column holds one datatype", da, db)}
if a.datatype != DataTypeString { if kindOf(b) < kindOf(a) {
read, other = b, da c.fix, c.other, c.want = b, a, da
} }
switch { message, ok := disagreements[[2]itemKind{kindOf(c.fix), kindOf(c.other)}]
case read.datatype != DataTypeString: if !ok {
return fmt.Errorf("its items hold %s and %s; a column holds one datatype", da, db) panic(invariant.Broken("disagreement has no message for item kinds %d and %d; two text items hold one datatype, so they never disagree", kindOf(c.fix), kindOf(c.other)))
case (&valueProof{}).columnItem(read).not[other] == "":
return fmt.Errorf("its items hold %s and %s; a column holds one datatype, so %s", da, db, typedAs(read, other))
} }
return fmt.Errorf("its items hold %s and %s; a column holds one datatype, so to read %q as text, %s", da, db, read.format, asText(read)) return message(c)
}
// clash is two items of one column holding different datatypes: other holds want; held says
// what each holds.
type clash struct {
fix, other *template
want DataType
held string
}
// disagreements is a clash's message by the kinds of its items, fix's first.
var disagreements = map[[2]itemKind]func(clash) error{
{kindDeclares, kindDeclares}: clash.bothDeclare,
{kindReads, kindReads}: clash.retypeRead,
{kindReads, kindDeclares}: clash.retypeRead,
{kindText, kindReads}: clash.textBesideRead,
{kindText, kindDeclares}: clash.declareText,
}
func (c clash) bothDeclare() error { return errors.New(c.held) }
// fits reports whether fix may hold want.
func (c clash) fits() bool { return (&valueProof{}).proveColumnItem(c.fix).Not[c.want] == "" }
func (c clash) retypeRead() error {
if c.fits() {
return fmt.Errorf("%s, so %s", c.held, typedAs(c.fix, c.want))
}
return fmt.Errorf("%s, so to read %q as text, %s", c.held, c.fix.format, typedAs(c.fix, DataTypeString))
}
func (c clash) textBesideRead() error {
if !c.fits() {
return fmt.Errorf(`item %q is not %s, the datatype item %q takes from the column it reads; to read that column as text, %s`, c.fix.format, datatype.Noun(c.want), c.other.format, typedAs(c.other, DataTypeString))
}
return c.declareText()
}
func (c clash) declareText() error {
if c.fix.fromString { // an object may carry a weight, which this spelling would drop
return fmt.Errorf(`item %q declares no datatype, and a column holds one; write it as {"format":%q,"datatype":%q}`, c.fix.format, c.fix.format, c.want)
}
return fmt.Errorf(`item %q declares no datatype beside one holding %s; a column holds one, so give it "datatype": %q`, c.fix.format, c.want, c.want)
} }
// typedAs names the spelling giving a column-read item datatype d, keeping the other keys an object // typedAs names the spelling giving a column-read item datatype d, keeping the other keys an object
@@ -199,11 +215,43 @@ func typedAs(t *template, d DataType) string {
return fmt.Sprintf(`give %q "datatype": %q`, t.format, d) return fmt.Sprintf(`give %q "datatype": %q`, t.format, d)
} }
// asText names the spelling that reads the column a column-read item reads as text, keeping the // columnRead is a record's column read by a format that only reads one reference or name,
// other keys an object item carries. // which is the column: it takes the column's datatype and null. category and field name it;
func asText(t *template) string { // category is "" for a column of the reading template's own record, which checkColumns reaches
if t.fromString { // on its own.
return fmt.Sprintf(`write {"format":"{text}","text":%q}`, t.format) type columnRead struct {
} a arm
return fmt.Sprintf(`set its "format" to "{text}" and add "text": %q`, t.format) category, field string
column node
}
// columnReadOf is the record's column t's format reads, where the format only reads one reference
// or name, and nil where it does not or t declares a string, which reads the column as text.
func columnReadOf(t *template, targets map[*nameBinding]nameTarget) *columnRead {
ops := t.compiled.ops
if t.datatype != nil && *t.datatype == DataTypeString || t.repeat != 1 || len(ops) != 1 || ops[0].Kind != grammar.PathRead || len(ops[0].arms) != 1 {
return nil
}
a := ops[0].arms[0]
var start node
var category string
var tail []string
switch b := a.named; {
case a.kind == namedRead && b.bindsField():
// {f as n}{n}: column f of the record binding n, in t's own category.
start, tail = b.binder, append(append([]string{splitArm(b.ref, nil).head}, targets[b].tail...), a.tail...)
case a.kind == namedRead:
// {/c as n}{n.x} or {/c.x as n}{n}: column x of category c.
start, category = targets[b].start, categoryOf(b.binder.refs.byName[b.ref].head)
tail = append(targets[b].tail[:len(targets[b].tail):len(targets[b].tail)], a.tail...)
case grammar.IsRef(a.head):
// {/c.x}: column x of category c.
start, category, tail = t.startOf(a.head), categoryOf(a.head), a.tail
default:
return nil
}
if target, isTemplate := start.(*template); isTemplate && target.isRecord && len(tail) == 1 {
return &columnRead{a: a, category: category, field: tail[0], column: target.fields[tail[0]]}
}
return nil
} }
+70 -24
View File
@@ -9,7 +9,7 @@ import (
"testing" "testing"
) )
func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) { func TestDatatypeSitsOnlyInAColumn(t *testing.T) {
for _, src := range []string{ for _, src := range []string{
`{"format":"","age":{"format":"{int(18,99)}","datatype":"integer"}}`, `{"format":"","age":{"format":"{int(18,99)}","datatype":"integer"}}`,
`{"format":"","n":{"format":"42","datatype":"integer"}}`, `{"format":"","n":{"format":"42","datatype":"integer"}}`,
@@ -18,21 +18,15 @@ func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) {
`{"format":"","age":[null,{"format":"{int(18,99)}","datatype":"integer","weight":9}]}`, `{"format":"","age":[null,{"format":"{int(18,99)}","datatype":"integer","weight":9}]}`,
`{"format":"","pick":[[null,"a"],"b"]}`, `{"format":"","pick":[[null,"a"],"b"]}`,
} { } {
if _, err := compile(parse(t, src)); err != nil { if _, err := resolved(t, src); err != nil {
t.Errorf("compile(%s) = %v, want a column to take a datatype and null", src, err) t.Errorf("compile(%s) = %v, want a column to take a datatype and null", src, err)
} }
} }
for src, want := range map[string]string{ for src, want := range map[string]string{
`{"format":"","n":{"format":"1","datatype":"int"}}`: `datatype takes "integer", "number" or "boolean", got "int"`, `{"format":"","n":{"format":"1","datatype":"int"}}`: `datatype takes "string", "integer", "number" or "boolean", got "int"`,
`{"format":"","n":{"format":"1","datatype":1}}`: "datatype must be a string", `{"format":"","n":{"format":"1","datatype":1}}`: "datatype must be a string, not a number",
`{"format":"{int(1,9)}","datatype":"integer"}`: "datatype only types a record column",
`[{"format":"1","datatype":"integer"},"x"]`: "datatype only types a record column",
`{"format":"{p}","p":{"format":"{n}","n":{"format":"1","datatype":"integer"}}}`: "datatype only types a record column",
`{"format":"{n}","repeat":2,"n":{"format":"1","datatype":"integer"}}`: "datatype only types a record column",
`null`: `so write ""`,
`{"format":"{p}","p":{"format":"{x}","x":[null,"a"]}}`: `so write ""`,
} { } {
if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) { if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("compile(%s) = %v, want an error containing %q", src, err, want) t.Errorf("compile(%s) = %v, want an error containing %q", src, err, want)
} }
} }
@@ -41,27 +35,30 @@ func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) {
func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) { func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
tree := map[string]string{ tree := map[string]string{
"cat": `[{"format":"{code}","code":"200"},{"format":"{code}","code":"2x","note":"n"}]`, "cat": `[{"format":"{code}","code":"200"},{"format":"{code}","code":"2x","note":"n"}]`,
"src": `{"format":"","code":[null,"200","2x"],"flag":{"format":"{b}","b":["true","false"],"datatype":"boolean"},"score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`, "src": `{"format":"","code":[null,"200","2x"],"flag":{"format":"{b}","b":["true","false"],"datatype":"boolean"},"ratio":{"format":"{float(0,9,0)}","datatype":"number"},"score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`,
} }
for _, c := range []struct{ name, column, want string }{ for _, c := range []struct{ name, column, want string }{
{"an item beside a typed one", `[{"format":"1","datatype":"integer"},"x"]`, `write it as {"format":"x","datatype":"integer"}`}, {"an item beside a typed one", `[{"format":"1","datatype":"integer"},"x"]`, `write it as {"format":"x","datatype":"integer"}`},
{"a weighted item beside a typed one", `[{"format":"1","datatype":"integer"},{"format":"2","weight":3}]`, `give it "datatype": "integer"`}, {"a weighted item beside a typed one", `[{"format":"1","datatype":"integer"},{"format":"2","weight":3}]`, `give it "datatype": "integer"`},
{"items of two datatypes", `[{"format":"1","datatype":"integer"},{"format":"true","datatype":"boolean"}]`, "a column holds one datatype"},
{"an item beside a typed column it reads", `["{/src.score}","5"]`, `write it as {"format":"5","datatype":"integer"}`}, {"an item beside a typed column it reads", `["{/src.score}","5"]`, `write it as {"format":"5","datatype":"integer"}`},
{"a typed item beside a string column it reads", `["{/src.code}",{"format":"1","datatype":"integer"}]`, `write it as {"format":"{/src.code}","datatype":"integer"}`}, {"a typed item beside a string column it reads", `["{/src.code}",{"format":"1","datatype":"integer"}]`, `write it as {"format":"{/src.code}","datatype":"integer"}`},
{"text beside a typed column it reads", `["{/src.score}","n/a"]`, `to read that column as text, write {"format":"{text}","text":"{/src.score}"}`}, {"text beside a typed column it reads", `["{/src.score}","n/a"]`, `to read that column as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
{"a datatype over a typed column", `{"format":"{/src.score}","datatype":"integer"}`, `{/src.score} takes datatype integer from the column it reads; drop "datatype"`},
{"a datatype a typed column's values reject", `{"format":"{/src.score}","datatype":"boolean"}`, "{int(1,9)} prints an integer, not a boolean"}, {"a datatype a typed column's values reject", `{"format":"{/src.score}","datatype":"boolean"}`, "{int(1,9)} prints an integer, not a boolean"},
{"two typed columns it reads", `["{/src.score}","{/src.flag}"]`, `so to read "{/src.score}" as text, write {"format":"{text}","text":"{/src.score}"}`}, {"two typed columns it reads", `["{/src.score}","{/src.flag}"]`, `so to read "{/src.score}" as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
{"a typed column read that holds the datatype declared beside it", `[{"format":"1.5","datatype":"number"},"{/src.score}"]`, `so write "{/src.score}" as {"format":"{/src.score}","datatype":"number"}`}, {"a typed column read that holds the datatype declared beside it", `[{"format":"1.5","datatype":"number"},"{/src.score}"]`, `so write "{/src.score}" as {"format":"{/src.score}","datatype":"number"}`},
{"text beside a weighted typed column read", `[{"format":"{/src.score}","weight":3},"n/a"]`, `to read that column as text, set its "format" to "{text}" and add "text": "{/src.score}"`}, {"a typed column read holding the datatype of another", `["{/src.ratio}","{/src.score}"]`, `so write "{/src.ratio}" as {"format":"{/src.ratio}","datatype":"integer"}`},
{"text before a typed column it reads", `["n/a","{/src.score}"]`, `to read that column as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
{"a weighted item beside a typed column it reads", `["{/src.score}",{"format":"5","weight":2}]`, `give it "datatype": "integer"`},
{"a typed column read before the datatype it holds", `["{/src.score}",{"format":"1.5","datatype":"number"}]`, `so write "{/src.score}" as {"format":"{/src.score}","datatype":"number"}`},
{"a typed column read beside a datatype it does not hold", `["{/src.score}",{"format":"true","datatype":"boolean"}]`, `so to read "{/src.score}" as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
{"text beside a weighted typed column read", `[{"format":"{/src.score}","weight":3},"n/a"]`, `to read that column as text, give "{/src.score}" "datatype": "string"`},
{"a value of the column it reads", `{"format":"{/src.code}","datatype":"integer"}`, `"2x" is not an integer`}, {"a value of the column it reads", `{"format":"{/src.code}","datatype":"integer"}`, `"2x" is not an integer`},
{"a null read into text", `{"format":"{x}","x":"{/src.score}","datatype":"integer"}`, "reads a null"}, {"a null read into text", `{"format":"{x}","x":"{/src.score}","datatype":"integer"}`, "reads a null"},
{"a sample with leading zeros", `{"format":"{digits(3)}","datatype":"integer"}`, "{digits(3)} prints text, not an integer"}, {"a sample with leading zeros", `{"format":"{digits(3)}","datatype":"integer"}`, "{digits(3)} prints text, not an integer"},
{"a fraction", `{"format":"{v}","v":["1","1.5"],"datatype":"integer"}`, `"1.5" is not an integer`}, {"a fraction", `{"format":"{v}","v":["1","1.5"],"datatype":"integer"}`, `"1.5" is not an integer`},
{"past int64", `{"format":"9223372036854775808","datatype":"integer"}`, "past the int64 range"}, {"past int64", `{"format":"9223372036854775808","datatype":"integer"}`, "past the int64 range"},
{"a float sample", `{"format":"{float(0,1,2)}","datatype":"integer"}`, "{float(0,1,2)} prints a number, not an integer"}, {"a float sample", `{"format":"{float(0,1,2)}","datatype":"integer"}`, "{float(0,1,2)} prints a number, not an integer"},
{"composed digits", `{"format":"1{digits(2)}","datatype":"integer"}`, "is not one value"}, {"composed digits", `{"format":"1{digits(2)}","datatype":"integer"}`, "is not one value; write one literal or one {float()}, {int()}, {seq()} or {calc()}, or read one"},
{"a sign before a sample", `{"format":"-{int(1,9)}","datatype":"integer"}`, "is not one value"}, {"a sign before a sample", `{"format":"-{int(1,9)}","datatype":"integer"}`, "is not one value"},
{"a repeat", `{"format":"{int(1,9)}","repeat":2,"separator":",","datatype":"integer"}`, "carries a repeat"}, {"a repeat", `{"format":"{int(1,9)}","repeat":2,"separator":",","datatype":"integer"}`, "carries a repeat"},
{"through a reference", `{"format":"{/cat.code}","datatype":"integer"}`, `"2x" is not an integer`}, {"through a reference", `{"format":"{/cat.code}","datatype":"integer"}`, `"2x" is not an integer`},
@@ -75,7 +72,7 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
{"a divisor that can be zero", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{int(0,9)}","datatype":"number"}`, "divides by b, which is not proven nonzero"}, {"a divisor that can be zero", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{int(0,9)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
{"an overflow", `{"format":"{calc(a * a)}","a":"{digits(200)}","datatype":"number"}`, "is not proven within 1e300"}, {"an overflow", `{"format":"{calc(a * a)}","a":"{digits(200)}","datatype":"number"}`, "is not proven within 1e300"},
{"a division in an integer column", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{int(1,9)}","datatype":"integer"}`, "prints a number, not an integer"}, {"a division in an integer column", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{int(1,9)}","datatype":"integer"}`, "prints a number, not an integer"},
{"a composed operand", `{"format":"{calc(n * 2)}","n":"{int(1,99)}.{digits(2)}","datatype":"number"}`, "{seq()}, {digits()} or {calc()}"}, {"a composed operand", `{"format":"{calc(n * 2)}","n":"{int(1,99)}.{digits(2)}","datatype":"number"}`, "one {digits()}, {float()}, {int()}, {seq()} or {calc()}"},
{"a divisor that prints as zero", `{"format":"{calc(1 / b, 2)}","b":"{float(4.9999999999999994e-79,5e-79,78)}","datatype":"number"}`, "divides by b, which is not proven nonzero"}, {"a divisor that prints as zero", `{"format":"{calc(1 / b, 2)}","b":"{float(4.9999999999999994e-79,5e-79,78)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
{"a divisor that rounds to zero", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{float(0.1,1,0)}","datatype":"number"}`, "divides by b, which is not proven nonzero"}, {"a divisor that rounds to zero", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{float(0.1,1,0)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
{"a zero among a divisor's literals", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":["0","5"],"datatype":"number"}`, "divides by b, which is not proven nonzero"}, {"a zero among a divisor's literals", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":["0","5"],"datatype":"number"}`, "divides by b, which is not proven nonzero"},
@@ -85,15 +82,13 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
{"an operand past the limit", `{"format":"{calc(a)}","a":"{digits(400)}","datatype":"number"}`, "is not proven within 1e300"}, {"an operand past the limit", `{"format":"{calc(a)}","a":"{digits(400)}","datatype":"number"}`, "is not proven within 1e300"},
{"a whole calc past int64", `{"format":"{calc(a * 2)}","a":"{seq()}","datatype":"integer"}`, "{calc(a * 2)} is not proven within int64"}, {"a whole calc past int64", `{"format":"{calc(a * 2)}","a":"{seq()}","datatype":"integer"}`, "{calc(a * 2)} is not proven within int64"},
{"a whole float past int64", `{"format":"{float(0,1e19,0)}","datatype":"integer"}`, "{float(0,1e19,0)} is not proven within int64"}, {"a whole float past int64", `{"format":"{float(0,1e19,0)}","datatype":"integer"}`, "{float(0,1e19,0)} is not proven within int64"},
{"a signed zero integer", `{"format":"-0","datatype":"integer"}`, `"-0" is zero written with a sign; write "0"`},
{"a signed zero number", `{"format":"-0.00","datatype":"number"}`, `"-0.00" is zero written with a sign; write "0.00"`},
} { } {
row := `{"format":"","col":` + c.column + `}` row := `{"format":"","col":` + c.column + `}`
files := map[string]string{"row": row} files := map[string]string{"row": row}
for name, body := range tree { for name, body := range tree {
files[name] = body files[name] = body
} }
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))) _, err := New(WithSeed(1), WithDataPath(writeData(t, files)))
if err == nil || !strings.Contains(err.Error(), c.want) { if err == nil || !strings.Contains(err.Error(), c.want) {
t.Errorf("%s: New = %v, want an error containing %q", c.name, err, c.want) t.Errorf("%s: New = %v, want an error containing %q", c.name, err, c.want)
} }
@@ -102,7 +97,19 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
t.Errorf("%s: NewTemplate = %v, want the inline template refused the same way", c.name, err) t.Errorf("%s: NewTemplate = %v, want the inline template refused the same way", c.name, err)
} }
} }
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{ for column, want := range map[string]string{
`[{"format":"1","datatype":"integer"},{"format":"true","datatype":"boolean"}]`: "its items hold integer and boolean; a column holds one datatype",
`[{"format":"1","datatype":"integer"},{"format":"1.5","datatype":"number"}]`: "its items hold integer and number; a column holds one datatype",
} {
row := `{"format":"","col":` + column + `}`
if _, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{"row": row}))); err == nil || !strings.HasSuffix(err.Error(), want) {
t.Errorf("items of two declared datatypes %s: New = %v, want an error ending %q", column, err, want)
}
if _, err := newGenerator(t, writeData(t, tree)).NewTemplate(row); err == nil || !strings.HasSuffix(err.Error(), want) {
t.Errorf("items of two declared datatypes %s: NewTemplate = %v, want an error ending %q", column, err, want)
}
}
_, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{
"a": `{"format":"","c":["{/b.x}",{"format":"1","datatype":"integer"}]}`, "a": `{"format":"","c":["{/b.x}",{"format":"1","datatype":"integer"}]}`,
"b": `{"format":"","x":["2",{"format":"1","datatype":"integer"}]}`, "b": `{"format":"","x":["2",{"format":"1","datatype":"integer"}]}`,
}))) })))
@@ -141,7 +148,7 @@ func TestDatatypeAcceptsAColumnThatAlwaysParses(t *testing.T) {
} { } {
row := `{"format":"","col":` + column + `}` row := `{"format":"","col":` + column + `}`
src := `{"format":"","code":["200","404"],"n":{"format":"{int(1,9)}","datatype":"integer"}}` src := `{"format":"","code":["200","404"],"n":{"format":"{int(1,9)}","datatype":"integer"}}`
f, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": cat, "row": row, "src": src})), WithSeed(1)) f, err := New(WithDataPath(writeData(t, map[string]string{"cat": cat, "row": row, "src": src})), WithSeed(1))
if err != nil { if err != nil {
t.Errorf("%s: New = %v, want it loaded", column, err) t.Errorf("%s: New = %v, want it loaded", column, err)
continue continue
@@ -204,3 +211,42 @@ func TestNullColumn(t *testing.T) {
t.Errorf("List() = %v, want the null column row.gone, which Fake accepts", f.List()) t.Errorf("List() = %v, want the null column row.gone, which Fake accepts", f.List())
} }
} }
func TestDisagreementRefusesTwoTextItems(t *testing.T) {
mustPanic(t, "two text items", func() { _ = disagreement(&template{}, DataTypeString, &template{}, DataTypeString) })
}
func TestASignedZeroIsZero(t *testing.T) {
f := newGenerator(t, writeData(t, map[string]string{"row": `{"format":"","i":{"format":"-0","datatype":"integer"},"n":{"format":"-0.00","datatype":"number"}}`}))
r, err := f.FakeRecord("row")
if err != nil {
t.Fatal(err)
}
if c := r.Columns(); c[0].Value != "-0" || c[1].Value != "-0.00" {
t.Errorf("columns = %+v, want each written as its format", c)
}
}
func TestADeclaredDatatypeOverAColumnRead(t *testing.T) {
f := newGenerator(t, writeData(t, map[string]string{
"src": `{"format":"","score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`,
"row": `{"format":"","same":{"format":"{/src.score}","datatype":"integer"},"text":{"format":"{/src.score}","datatype":"string"}}`,
}), WithSeed(1))
empty := 0
for i := 0; i < 50; i++ {
r, err := f.FakeRecord("row")
if err != nil {
t.Fatal(err)
}
same, text := r.Columns()[0], r.Columns()[1]
if same.DataType != DataTypeInteger || text.DataType != DataTypeString || text.Null {
t.Fatalf("columns = %+v, want an integer column restated and a string column never null", r.Columns())
}
if text.Value == "" {
empty++
}
}
if empty == 0 {
t.Error(`50 records never read a null as "", want the string column to render one so`)
}
}
+46
View File
@@ -0,0 +1,46 @@
// Package fejkdata generates locale-aware fake data for tests and fixtures from JSON templates.
//
// Data lives in JSON, not in Go, and comes in modules: a Go package exporting an fs.FS,
// or a folder. A generator loads the modules you name, such as data.Modules(), every
// module fejkdata ships, from github.com/larvit/fejkdata/data. Their folders and files
// become a dot-path namespace, and you generate values by path:
//
// f, _ := fejkdata.New(fejkdata.WithDataFS(data.Modules()...), fejkdata.WithSeed(42))
// f.Fake("sv_SE.address") // "Trekungagatan 43\n442 42 Kungälv"
// f.Fake("sv_SE.address.locality") // "Umeå": a second call draws afresh
//
// Several sources merge in order, the last winning a name clash, so custom data
// layers over the built-ins. The JSON template format is documented in the [README].
//
// [README]: https://github.com/larvit/fejkdata#readme
package fejkdata
// Vocabulary
//
// - draw — one item taken from a choice, or one row from a table: `drawItem`,
// `drawThroughChoices`, `renderEnv.drawRowOf`. Every {…} draws afresh, bar a read of a name.
// - expansion — one render of one format: `expand`.
// - render — one walk of the tree from an entry point, over one `frameStack`:
// `Generator.Fake`, `FakeRecord`, `Template.Fake` and `RecordTemplate.Fake`
// start one each, and `FakeStruct` one per record.
// - env — what a render reads names and rows through: `renderEnv`.
// - scope — where a name is bound: a category, or one repeat iteration: `nameScope`.
// - named pick — the draw a token such as {/person as p} binds to a name, drawn
// on its first read and kept while its scope renders: `nameBinding`,
// `namedPick`, `pickFrame`.
// - level, pick key, addressed key — `pickKey`.
// - memo — what a named pick keeps its draws in, the variant drawn at each level
// and the value each path read: `drawMemo`.
// - pin — fixing which row of a table one path, or one named pick, uses:
// `pinSet`, `namedPick.pins`, `table.followStep`.
// - link — a table's tie to its parent table, each row to its parent row:
// `table.linkParent`.
// - family — a table and every table reaching it through a chain of parent
// links.
// - resolve — tying a reference or a name to what it names in the assembled tree:
// `resolveTemplates`, `resolveRefs`.
// - head — the string a read starts from: a field or a name as written, or a
// reference's category path with its sigil, /sv_SE.address: `arm.head`. The node it names is
// the read's start: `template.startOf`, `nameTarget.start`.
// - fence — a load-time check, run over each category, template or struct as it
// loads or compiles, so rendering a compiled tree cannot fail.
+286
View File
@@ -0,0 +1,286 @@
package fejkdata
import (
"go/ast"
"go/build"
"go/parser"
"go/token"
"os"
"regexp"
"slices"
"strconv"
"strings"
"testing"
)
// Every backtick in the Vocabulary is a symbol.
var backticked = regexp.MustCompile("`([^`\n]+)`")
// TestVocabularyNamesDeclaredSymbols holds a renamed unit to its Vocabulary entry.
func TestVocabularyNamesDeclaredSymbols(t *testing.T) {
_, files := sourceFiles(t)
declared := declaredSymbols(files)
if len(namesDeclared(t, "the Vocabulary", vocabulary(t, files), declared)) == 0 {
t.Fatal("the Vocabulary names no symbol, so nothing holds a renamed unit to its entry")
}
}
// namesDeclared is every symbol text backticks, failing for each one the package
// declares nowhere.
func namesDeclared(t *testing.T, where, text string, declared map[string]bool) []string {
t.Helper()
var named []string
for _, m := range backticked.FindAllStringSubmatch(text, -1) {
if !declared[m[1]] {
t.Errorf("%s names %s, which the package declares nowhere", where, m[1])
}
named = append(named, m[1])
}
return named
}
// namesAPass fails unless one symbol text backticks is a function: a field names
// what a pass reads or writes, never the pass.
func namesAPass(t *testing.T, where, text string, declared, funcs map[string]bool) {
t.Helper()
named := namesDeclared(t, where, text, declared)
if !slices.ContainsFunc(named, func(n string) bool { return funcs[n] }) {
t.Errorf("%s names no function, so it names no pass", where)
}
}
func packageDir(t *testing.T) *build.Package {
t.Helper()
pkg, err := build.ImportDir(".", 0)
if err != nil {
t.Fatalf("read the package directory: %v", err)
}
return pkg
}
func sourceFiles(t *testing.T) (*token.FileSet, []*ast.File) {
t.Helper()
return parseFiles(t, packageDir(t).GoFiles)
}
func parseFiles(t *testing.T, names []string) (*token.FileSet, []*ast.File) {
t.Helper()
fset := token.NewFileSet()
var files []*ast.File
for _, name := range names {
f, err := parser.ParseFile(fset, name, nil, parser.ParseComments)
if err != nil {
t.Fatalf("parse %s: %v", name, err)
}
files = append(files, f)
}
return fset, files
}
func vocabulary(t *testing.T, files []*ast.File) string {
t.Helper()
var sections strings.Builder
for _, f := range files {
for _, c := range f.Comments {
section, found := strings.CutPrefix(c.Text(), "Vocabulary\n\n")
if found && c.Pos() > f.Package {
sections.WriteString(section)
}
}
}
if sections.Len() == 0 {
t.Fatal("no comment below a package clause opens with a Vocabulary heading, so the words the package is written in are defined nowhere a reader of the code meets them")
}
return sections.String()
}
// declaredSymbols is every name the package declares at the top level, a method and
// a struct field keyed under its type: namedPick.pins, tableRead.landsRow.
func declaredSymbols(files []*ast.File) map[string]bool {
names := declaredFuncs(files)
for _, f := range files {
for _, decl := range f.Decls {
if d, isGen := decl.(*ast.GenDecl); isGen {
addSpecs(names, d)
}
}
}
return names
}
// declaredFuncs is every function the package declares, a method keyed under its
// type: template.fixedText.
func declaredFuncs(files []*ast.File) map[string]bool {
names := map[string]bool{}
for _, f := range files {
for _, decl := range f.Decls {
d, isFunc := decl.(*ast.FuncDecl)
if !isFunc {
continue
}
names[d.Name.Name] = true
if recv := receiverType(d.Recv); recv != "" {
names[recv+"."+d.Name.Name] = true
}
}
}
return names
}
func addSpecs(names map[string]bool, d *ast.GenDecl) {
for _, spec := range d.Specs {
switch s := spec.(type) {
case *ast.TypeSpec:
names[s.Name.Name] = true
addFields(names, s)
case *ast.ValueSpec:
for _, id := range s.Names {
names[id.Name] = true
}
}
}
}
func addFields(names map[string]bool, s *ast.TypeSpec) {
st, isStruct := s.Type.(*ast.StructType)
if !isStruct {
return
}
for _, field := range st.Fields.List {
for _, id := range field.Names {
names[s.Name.Name+"."+id.Name] = true
}
}
}
func receiverType(recv *ast.FieldList) string {
if recv == nil || len(recv.List) == 0 {
return ""
}
e := recv.List[0].Type
if star, isPointer := e.(*ast.StarExpr); isPointer {
e = star.X
}
id, isIdent := e.(*ast.Ident)
if !isIdent {
return ""
}
return id.Name
}
const fillHeader = "Filled by "
// TestTemplateFieldGroupsNameADeclaredPass holds a renamed pass to the group it
// fills, and to the template doc.
func TestTemplateFieldGroupsNameADeclaredPass(t *testing.T) {
fset, files := sourceFiles(t)
declared, funcs := declaredSymbols(files), declaredFuncs(files)
doc, st, headers := templateStruct(t, fset, files)
namesAPass(t, "the template doc", doc, declared, funcs)
for _, field := range st.Fields.List {
if len(field.Names) == 0 {
t.Error("an embedded field of template sits under no header, so nothing says which pass fills it")
}
for _, id := range field.Names {
header := headerAbove(headers, id.Pos())
if header == "" {
t.Errorf("template.%s sits under no %q header, so nothing says which pass fills it", id.Name, fillHeader)
continue
}
namesAPass(t, "the header above template."+id.Name, header, declared, funcs)
}
}
}
// templateStruct is the template struct, its doc, and the fill headers standing
// between its fields, in source order.
func templateStruct(t *testing.T, fset *token.FileSet, files []*ast.File) (string, *ast.StructType, []*ast.CommentGroup) {
t.Helper()
for _, f := range files {
for _, decl := range f.Decls {
d, isGen := decl.(*ast.GenDecl)
if !isGen {
continue
}
for _, spec := range d.Specs {
s, isType := spec.(*ast.TypeSpec)
if !isType || s.Name.Name != "template" {
continue
}
st, isStruct := s.Type.(*ast.StructType)
if !isStruct {
t.Fatal("template is not a struct")
}
return d.Doc.Text() + s.Doc.Text(), st, fillHeaders(fset, f, st)
}
}
}
t.Fatal("the package declares no template struct")
return "", nil, nil
}
// fillHeaders skips a header trailing a field: it names that field, never the
// fields below it.
func fillHeaders(fset *token.FileSet, f *ast.File, st *ast.StructType) []*ast.CommentGroup {
trailing := map[int]bool{}
for _, field := range st.Fields.List {
trailing[fset.Position(field.End()).Line] = true
}
var found []*ast.CommentGroup
for _, c := range f.Comments {
if c.Pos() < st.Pos() || c.End() > st.End() || trailing[fset.Position(c.Pos()).Line] {
continue
}
if strings.HasPrefix(c.Text(), fillHeader) {
found = append(found, c)
}
}
return found
}
// headerAbove is the last fill header standing above pos.
func headerAbove(headers []*ast.CommentGroup, pos token.Pos) string {
text := ""
for _, c := range headers {
if c.Pos() < pos {
text = c.Text()
}
}
return text
}
var (
printedCall = regexp.MustCompile(`(?m)^.*Fake\w*\(.*//\s*".*$`)
printedFake = regexp.MustCompile(`\.Fake\("([^"]+)"\)\s*//\s*("(?:[^"\\]|\\.)*")`)
)
// TestGoExamplesPrintWhatTheyRender replays each Go example's printed Fake calls on seed 42.
func TestGoExamplesPrintWhatTheyRender(t *testing.T) {
for _, name := range []string{"doc.go", "README.md"} {
src, err := os.ReadFile(name)
if err != nil {
t.Fatal(err)
}
lines := printedCall.FindAllString(string(src), -1)
if lines == nil {
t.Errorf("%s prints no seeded Fake, so nothing holds its example to a render", name)
}
f, err := New(withShipped(), WithSeed(42))
if err != nil {
t.Fatal(err)
}
for _, line := range lines {
c := printedFake.FindStringSubmatch(line)
if c == nil {
t.Fatalf("%s prints a value this test cannot replay; spell it f.Fake(\"path\") // \"value\": %s", name, line)
}
want, err := strconv.Unquote(c[2])
if err != nil {
t.Fatalf("%s prints %s: %v", name, c[2], err)
}
if got := fake(t, f, c[1]); got != want {
t.Errorf("%s prints Fake(%q) = %q for seed 42, it renders %q", name, c[1], want, got)
}
}
}
}
+722
View File
@@ -0,0 +1,722 @@
# Comprehension history
## 2026-10-09T08:59:19Z, PR #190 at 17f732a, against ee4b40b
Ruling: better
| Seat | Vote |
|---|---|
| Mid A | better |
| Inherited architect | better |
Mid A, decided by:
- `pick.go:9` `pickKey`: the definition sits on the type, beside `levelKeys`, `addressedKeys`, `keptInPick` and `readUnder`, where four files held them
- `render.go:155` `readField`: one named predicate, `keptInPick`, states the whole keep rule
- `datatype.go:217` `columnRead`: beside the code that reads it
Inherited architect, decided by:
- `pick.go:9` `pickKey`: the 3am path for a named pick now ends in one file, with its worked example
- `render.go:155` `readField`: `keptInPick` holds its own nil and reference checks
- `path.go:72` `compiledPath`: the field no longer reuses the vocabulary word level
## 2026-10-09T09:24:31Z, PR #190 at 65a32db, against ee4b40b
Ruling: better
| Seat | Vote |
|---|---|
| Mid A | better |
| Inherited architect | better |
Mid A, decided by:
- `pick.go:9` `pickKey`: the definitions sit on the type, beside the functions that build and read the keys
- `datatype.go:217` `columnRead`: beside `itemDatatype` and `checkColumns`, which consume it
- `node.go:70` `template.readsColumn`: names the four readers that change together
Inherited architect, decided by:
- `pick.go:9` `pickKey`: a pick-key bug needs one file, where it needed four and the vocabulary
- `render.go:155` `readField`: one named predicate, `keptInPick`, states the keep rule
- `path.go:72` `compiledPath`: `headSpans` is the costliest new read in the diff
## 2026-10-09T09:30:15Z, PR #190 at d9f0fc9, against ee4b40b
Ruling: better
| Seat | Vote |
|---|---|
| Mid A | better |
| Inherited architect | better |
Mid A, decided by:
- `pick.go:9` `pickKey`: the definitions and the code building and reading the keys share one file
- `render.go:155` `readField`: `keptInPick` holds the whole keep rule and its own nil check
- `path.go:74` `compiledPath`: `headSpans` and the bare 0 its callers pass cost a second read
Inherited architect, decided by:
- `pick.go:9` `pickKey`: the 3am mechanism reads in one file, in reading order
- `datatype.go:217` `columnRead`: beside its consumers, and `node.go:70` names its four readers
- `path.go:119` `compileStep`: six parameters, the one place that reads harder
## 2026-10-09T10:16:38Z, PR #193 at c27e8b7, against d74c0ed
Ruling: better
| Seat | Vote |
|---|---|
| Mid A | better |
| Inherited architect | better |
Mid A, decided by:
- `data-import/geo.py:75` `fence`: one named step states the locality row and drops childless parents level by level, where `with_streets` pruned localities alone
- `data-import/geo-us.py:174` `main`: the county and region pruning no longer hides inside the `tsv.write` comprehensions
- `data-import/geo-se.py:186` `main`: builds its rows before the fence, so it has the US script's shape
Inherited architect, decided by:
- `data-import/geo.py:75` `fence`: the loader's parent-has-a-child rule is named in the import code and applied in one place at all three levels
- `data-import/geo-us.py:174` `main`: one `fence` call before any write replaces filters buried in the writes
- `data-import/geo-se.py:186` `main`: both scripts read alike, build rows, fence, write
## 2026-10-09T10:29:19Z, PR #193 at 8433689, against d74c0ed
Ruling: better
| Seat | Vote |
|---|---|
| Mid A | better |
| Inherited architect | better |
Mid A, decided by:
- `data-import/geo.py:75` `fence`: the parent-has-a-child rule lives in one function whose docstring states the rule, each argument's shape and what it returns
- `data-import/geo-us.py:174` `main`: `zctas` is built once and fenced, and every write reads from `kept` with no filter of its own
- `data-import/geo-us.py:128` `zcta_localities`: the name says it returns a ZCTA-to-place map, where `postal_codes` pointed at the wrong output
Inherited architect, decided by:
- `data-import/geo.py:75` `fence`: one unit owns the chain from locality to municipality to region, where the base spread it over `with_streets` and both `main`s
- `data-import/geo-us.py:128` `zcta_localities`: the name and docstring say what it maps to
- `data-import/geo-se.py:181` `main`: `region_names` and `municipality_names` say they map codes to names, and the rows are built once before the fence
## 2026-10-07, scoring run at ad59967
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mean | 7.00 | 5.88 | 6.50 | 6.13 | 6.38 |
## 2026-10-09T11:04:29Z, PR #195 at b9d1af6
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Junior A | 7.5 | 6.5 | 7 | 7 | 7 |
| Mid A | 7.5 | 6 | 6.5 | 7 | 6.5 |
| Maintainability senior | 7.5 | 6 | 6.5 | 6.5 | 6.5 |
| Inherited architect | 7 | 5.5 | 6 | 7 | 6.5 |
| Mean | 7.38 | 6.00 | 6.50 | 6.88 | 6.63 |
Junior A, hardest first:
- `pick.go:70` `keptInPick`: a level, a first-level key, that key moved under `env.pickAt`, and the name's addressed set, held at once
- `namefence.go:76` `refuseTwiceDrawnIn`: mirrors the keep rule in `keptInPick`, and nothing ties the two in code
Mid A, hardest first:
- `pick.go:79` `readUnder`: a key built at load in `addressedKeys` must equal one built at render through `pickKey.under`, and a mismatch is silent
- `node.go:70` `readsColumn`: four functions in three files must change together
Maintainability senior, hardest first:
- `render.go:146` `readField`: whether a read is kept in a pick rests on string keys from three places and on `env.pick` cleared at the right moment
- `path.go:190` `drawSteps`: a nil `pins` or `memo` names the mode, and `climbed` carries state across iterations
Inherited architect, would restructure first:
- `pick.go:179` `readName`: keeping one named pick needs `name.go`, `pick.go`, `env.go`, `render.go`, `path.go` and `record.go` held at once
- `node.go:52` `template`: filled in four phases by four functions
## 2026-10-09T11:09:34Z, PR #195 at 344023d, against 1c342bc
Ruling: better
| Seat | Vote |
|---|---|
| Mid A | better |
| Inherited architect | better |
Mid A, decided by:
- `shipped.go:61` `loadCallerPath`: `Fake` and `FakeRecord` get their segments from the one call that loads them, where each repeated the split and the load
- `node.go:30` `choice`: the `shared` comment says what the field counts, where it claimed `List` reads it
- `data-import/tsv.py:7` `write`: its check and message match the loader's one-row rule
Inherited architect, decided by:
- `shipped.go:61` `loadCallerPath`: the load-before-walk order is held by data flow, where a comment on `loadShippedAt` stated it
- `node.go:30` `choice`: the `shared` comment no longer sends a `--list` bug to the wrong field
- `README.md:163` `## Data`: the opening names the `geo` trees the section documents
## 2026-10-09T16:48:45Z, PR #201 at c3ef728, against 2652779
Ruling: same
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | same | same | same | same | same |
| Inherited architect | same | same | same | same | same |
## 2026-10-09T16:54:01Z, PR #201 at 8b5a972, against 2652779
Ruling: same
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | same | same | same | same | same |
| Inherited architect | same | same | same | same | same |
## 2026-10-09T17:21:57Z, PR #202 at 65b2de8, against 2652779
Ruling: worse
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | same | worse | same | same | worse |
| Inherited architect | same | worse | worse | worse | worse |
Mid A, decided by:
- Locality, Overall: `data.go:13` `loadSources`: one pass mixes lazy and eager loading through before/after snapshots, a `provides` closure and a reverse graph search, and the skip of the first source is unexplained
- Locality: `data.go:107` `mergeFolder`: a category must sit in one of two maps, `children` or `unloaded`, kept by deletes in four functions across two files
- Locality: `internal/datafiles/datafiles.go:83` `embeddedManifests`: a process-wide cache in a package that was otherwise pure
Inherited architect, decided by:
- Locality, Self-sufficiency: `data.go:13` `loadSources`: nothing says why replaced paths drag their readers in, or what the `!after[p]` case covers
- Locality: `index.go:113` `loadReached`: its rollback claims the next call fails the same way, which holds only while no parent table was linked to a rolled-back child
- Shape: `index.go:159` `reaching`: four near-synonyms, `reaching`, `reachedFrom`, `loadReached` and `referenced`, and `referenced` and `categoryUnder` return loaded categories too
- Shape: `data.go:40` `place`: shares its name with `compileInto`'s parameter, and returns a closure answering one question two ways
- Locality: `internal/datafiles/datafiles.go:83` `embeddedManifests`: hidden global state for a small saving
## 2026-10-09T17:32:44Z, PR #202 at fdc6a04, against 2652779
Ruling: worse
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | same | worse | same | same | worse |
| Inherited architect | same | worse | worse | worse | worse |
Mid A, decided by:
- Locality, Overall: `index.go:175` `loadReached`: one loop loads, puts back on failure, checks each entry before linking and again after the pipeline, and its rollback rests on an ordering invariant its comment argues
- Locality: `data.go:13` `loadSources`: whether a category loads in New means holding every source and every index entry together, through `standing` snapshots and `dependents`
- Shape: `index.go:36` `categoryAt`: a near-synonym of `categorySite`, beside `dependencies` and `dependents`, which differ by two letters and run in opposite directions
Inherited architect, decided by:
- Locality, Overall: `index.go:175` `loadReached`: five jobs in one queue loop, with a cross-call ordering invariant to hold
- Locality, Shape: `data.go:13` `loadSources`: one source can make another load early through identity-compared snapshots and a reverse graph over index entries
- Shape: `index.go:36` `categoryAt`: two near-synonym types for where a category sits, and three near-identical function names
- Self-sufficiency: `index.go:248` `staleEntry`: its note on linking cannot be read without knowing the pipeline, and `config.load` points at a paragraph-long decision anchor
## 2026-10-09T17:39:40Z, PR #202 at 0bdb100, against 2652779
Ruling: worse
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | same | worse | worse | same | worse |
| Inherited architect | same | worse | worse | same | worse |
Mid A, decided by:
- Locality, Overall: `data.go:13` `loadSources`: whether an indexed category loads in New depends on source order and what other sources hold, through before-and-after snapshots
- Locality: `index.go:119` `affectedBy`: trusts the manifest's `reads` to match the templates, with nothing beside the code checking it
- Locality, Shape: `index.go:174` `loadReached`: a rollback batch to reason about, with names that say less than `unloadedReads` and `linkedTables` did
- Shape: `index.go:16` `ErrLoad`: a public error every entry point depends on, in a file named for the index
- Shape: `internal/datafiles/datafiles.go:103` `decodeManifest`: a hand-written strict decoder doubles the package's responsibilities
Inherited architect, decided by:
- Locality, Overall: `data.go:13` `loadSources`: New's load set follows dependencies forward from templates and backward from the manifest's `reads`, and a stale `reads` leaves a broken reader for its first reach
- Locality: `index.go:201` `batch.load`: the unloaded entry is deleted three calls away, and `putBack` reverses it
- Locality, Shape: `data.go:62` `standing`: a `map[string]any` of nodes and entries compared by identity
- Shape: `index.go:1` `index.go`: fifteen units under generic names in a file named for the index, `ErrLoad` among them
- Shape: `internal/datafiles/datafiles.go:103` `decodeManifest`: a strict JSON decoder shares the tree walker's file with no seam
## 2026-10-10T09:09:41Z, PR #202 at d34cc42, against 2652779
Ruling: better
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | same | better | better | same | better |
| Inherited architect | same | better | same | same | better |
Mid A, decided by:
- Locality, Overall: `data.go:12` `loadSources`: one path, each source indexed or walked then one `loadReached`, where `config.load` switched between a lazy and an eager mode
- Locality: `index.go:107` `loadReached`: a failed load puts back what it loaded and returns an error, where `loadShipped` panicked partway through
- Locality, Shape: `node.go:31` `folder.put`: two methods keep a name in `children` or `unloaded`, never both
- Shape: `index.go:29` `categoryAt`: `shipped.go` is only the embed, and `categorySite` embeds `categoryAt` in place of a duplicated `unloadedCategory`
Inherited architect, decided by:
- Locality: `data.go:12` `loadSources`: one load path, where `config.load` hid a mode switch between lazy and eager loading
- Locality: `index.go:107` `loadReached`: returns a load error, and names a manifest entry drifted from its table, where `loadShipped` panicked on a hand-regenerated Go file
- Locality: `node.go:23` `folder`: the children-or-unloaded rule stated once and kept by two methods
## 2026-10-10T09:21:22Z, PR #202 at b77ec94, against 2652779
Ruling: better
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | same | better | same | same | better |
| Inherited architect | same | worse | same | same | worse |
| Maintainability senior | same | better | same | better | better |
| Junior A | same | better | better | same | better |
Mid A, decided by:
- Locality, Overall: `node.go:31` `folder.put`: two methods keep a name in `children` or `unloaded`, where `unloadedCategory.load` kept it by a separate delete
- Locality, Overall: `data.go:22` `addSource`: each source's own manifest decides how it loads, where adding any data path switched the whole shipped set to eager loading
Inherited architect, decided by:
- Locality, Overall: `index.go:109` `loadReached`: one function serves New and first reach, rolls back through `putBack`, and rests on facts held elsewhere
- Locality: `index.go:22` `placeIndex`: how sources merge depends on their order and on which kind each is, and the children-or-unloaded rule holds only while `put` is the one writer
Maintainability senior, decided by:
- Locality, Overall: `data.go:12` `loadSources`: every source goes through one path, where `config.load` branched between two load paths of different eagerness
- Locality: `index.go:15` `indexed`: an unloaded entry carries its own source, where `unloadedCategory.load` was wired to the global `shippedSource`
- Self-sufficiency, Overall: `internal/datafiles/manifest.go:56` `decodeManifest`: the index is data checked strictly, where `shippedEntry` told the reader to empty a generated literal by hand
Junior A, decided by:
- Locality, Shape: `node.go:23` `folder`: `put` and `putUnloaded` keep a name in one map, where `compileInto` and `unloadedCategory.load` kept it as a pair
- Locality, Shape: `data.go:12` `loadSources`: one load path, where `config.load` forked and `loadShipped` was a second, panic-only pipeline
- Shape: `data.go:81` `categorySite`: embeds `categoryAt`, where `unloadedCategory` repeated its fields
## 2026-10-10T11:11:16Z, PR #203 at 421bb49, against bf1becb
Ruling: worse
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | same | worse | worse | same | worse |
| Inherited architect | same | worse | worse | same | worse |
Mid A, decided by:
- Locality, Overall: `node.go:28` `folder.reads`: a third per-name map beside children and unloaded with no invariant tying it to them; every placement site must remember a separate setReads call (`data.go:36` addSource, `data.go:76` mergeFolder, `index.go:27` placeIndex)
- Locality, Shape: `resolve.go:96` `withReads`: the manifest's reads travels through folder, categorySite.reads and templateSite.category to feed one error clause, four files to hold for one sentence of error text
- Shape: `internal/datafiles/manifest.go:22` `Manifest.Reads`: reads as a load dependency the loader acts on, though it only feeds error text, and "reads" already means a template reading a name or column
- Shape: `data/geo/SE/fs.go:2` `SE`: package names SE, US, en_US, sv_SE; in Modules SE.FS sits beside sv_SE.FS with nothing saying which is geography
- Locality: `fejkdata.go:94` `New`: loads exactly what the options name, easier than the hidden config{shipped: true} default, though config.given brings back a smaller flag
Inherited architect, decided by:
- Locality, Shape, Overall: `node.go:27` `folder.reads`: a third per-name map that put and putUnloaded leave alone, so addSource, mergeFolder and placeIndex must each call setReads, and batch.load relies on placeIndex having done it; action at a distance for one error suffix
- Locality: `resolve.go:94` `templateSite.withReads`: reads travels folder, categorySite.reads, a *categorySite per template, then withReads, four hops across three files
- Shape: `data/geo/SE/fs.go:2` `SE`: bare package names SE and US do not say geography at WithDataFS(sv_SE.FS, SE.FS); the namespace-doubling rule lives only in README and test code
- Shape, Self-sufficiency: `internal/datafiles/manifest.go:22` `Manifest.Reads`: suggests it drives loading though it only feeds withReads; "by default" needs README goal 9.2
- Locality, Navigation: `data/modules.go:17` `Modules`: the hidden shippedFS default is gone; Modules lists the modules, errNoData names the fix, and the CLI loads data.Modules() in plain sight
- Locality: `fejkdata.go:40` `config.given`: WithDataFS() with no arguments differs from no option, learnt only from its comment
## 2026-10-10T11:24:33Z, PR #203 at 70e552c, against bf1becb
Ruling: same
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | same | better | worse | same | same |
| Inherited architect | worse | worse | better | same | worse |
| Maintainability senior | worse | same | worse | same | worse |
| Junior A | same | worse | same | same | same |
Mid A, decided by:
- Locality: `fejkdata.go:94` `New`: loads only the modules its options name or fails with errNoData; the base silently added the embedded shippedSource in config.load
- Shape: `internal/datafiles/manifest.go:23` `Manifest.Reads`: Reads and categorySite.defaultModules suggest a loaded dependency, though only a hint reads them, and "by default" is never set against anything
- Locality, Shape: `data.go:12` `loadSources`: default modules reach a category by two routes, the walked map and indexed.defaultModules, and templateSite gains a pointer only to carry them
- Shape: `data/modules.go:18` `Modules`: package names SE, US, sv_SE, en_US are unidiomatic, and every embedded path repeats itself
Inherited architect, decided by:
- Locality, Overall: `data.go:12` `loadSources`: addSource fills a side map keyed by path that loadSources copies onto sites after the merge, so merge order and clashes must be reasoned about on the core load path
- Locality, Shape: `data.go:92` `categorySite.defaultModules`: the manifest's module list passes four carriers, set at two distant places and not by siteIn, so a future caller resolving siteIn sites gets a false "names no module" message
- Navigation: `data/geo/SE/geo/SE/address.json` `geo.SE.address`: every shipped file sits under a stuttered path
- Shape: `fejkdata.go:65` `WithDataFS`: the root no longer embeds data, datafiles.Source drops base, and data sits behind data/modules.go and five small packages
- Navigation, Shape: `internal/datafiles/manifest.go:23` `Manifest.Reads`: import paths that look resolved though nothing resolves them, generated by a test reached only through go:generate
Maintainability senior, decided by:
- Navigation, Overall: `data/geo/SE/geo/SE/address.json` `geo.SE.address`: the path repeats the namespace inside the module, one extra hop from category to file, two for geo
- Shape: `internal/datafiles/manifest.go:21` `Manifest.Reads`: reads sounds like a loaded dependency, and the same list is also defaultModules and walked
- Navigation, Shape: `data/modules.go:15` `go:generate`: regenerating a module's manifest runs a test one package up
- Shape: `data.go:20` `loadSources`: defaultModules reaches categorySite by two routes, and siteIn leaves it empty
Junior A, decided by:
- Locality: `data.go:12` `loadSources`: default modules pass through four files, an out-parameter map patched onto sites after the merge for walked sources and indexed.defaultModules copied in batch.load for indexed ones
- Locality: `fejkdata.go:94` `New`: reads straight through with no hidden default source, which only partly offsets the above
## 2026-10-11T09:52:08Z, PR #203 at 4c4bbcc, against bf1becb
Ruling: worse
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | same | same | worse | same | worse |
| Inherited architect | worse | same | worse | same | worse |
Mid A, decided by:
- Shape, Overall: `data/geo/SE/fs.go:2` `SE`: each module embeds a tree repeating its own name, and the package names SE, US, en_US, sv_SE are unidiomatic; five thin packages plus a list replace one embed
- Shape: `reference.go:124` `withDefaultModules`: "default modules" suggests modules that load by default, though nothing loads them; "by default" is spelled out only in the README
- Locality, Shape: `inline.go:40` `NewTemplate`: errors are decorated twice, by hand, at four entry points
- Locality: `fejkdata.go:101` `New`: the hidden shipped default is gone, offsetting the above
Inherited architect, decided by:
- Navigation, Overall: `data/sv_SE/fs.go:8` `FS`: every data file sits under a doubled folder, data/sv_SE/sv_SE/, data/geo/SE/geo/SE/, a stop at each hop of the 3am path
- Navigation: `data/modules.go:15` `go:generate`: the module list is written twice and the manifest writer lives in a root test run from the data package
- Shape: `data/geo/SE/fs.go:2` `SE`: a package named SE reads as Sweden beside sv_SE, and "module" means a Go package here
- Shape: `reference.go:126` `withDefaultModules`: one concept spelled reads, defaultModules and "read by default", the last defined only in the README
- Locality: `fejkdata.go:114` `New`: four call sites apply loadError, withDefaultModules and noDataNote in a fixed order, offset by the hidden shipped source being gone
## 2026-10-11T10:44:13Z, PR #203 at 36b5062, against bf1becb
Ruling: worse
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | worse | worse | worse | same | worse |
| Inherited architect | worse | worse | same | worse | worse |
Mid A, decided by:
- Locality, Shape, Overall: `index.go:110` `loadReached`: a reads parameter runs through four layers only to add a note to an error, and reads already means a template's reads
- Navigation: `internal/datafiles/datafiles.go:36` `Source.Mounted`: a shipped module's broken file is labelled geo/SE/address.json, a path that no longer exists
- Locality: `internal/datafiles/datafiles.go:77` `Source.Walk`: Walk adds the mount and Load removes it, two forms of a folder path in one type
- Locality: `fejkdata.go:68` `WithDataFS`: the shipped data no longer loads silently, a partial offset
- Shape: `reference.go:127` `withMissingModules`: a typo in manifest-less data still gets a note about modules
Inherited architect, decided by:
- Locality, Overall: `index.go:110` `loadReached`: reads threaded through six signatures for one error note
- Navigation: `internal/datafiles/datafiles.go:36` `Source.Mounted`: errors name sv_SE/address.json, a path that does not exist, where the base named the real data/sv_SE/address.json
- Navigation, Self-sufficiency: `data/svse/.fejkdata.json:26` `namespace`: a namespace no longer mirrors its folder, so the path alone no longer finds the file
- Self-sufficiency: `internal/datafiles/manifest.go:23` `Manifest.Reads`: its comment and withMissingModules' only make sense after the README
- Locality: `internal/datafiles/datafiles.go:77` `Source.Walk`: the mount is translated in opposite directions by Walk and Load
## 2026-10-11T11:30:01Z, PR #203 at 2a4694b, against bf1becb
Ruling: worse
Fell: Navigation. Under the floor of 7.0: Navigation 6.88, so the same ruling counts as worse.
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | worse | better | same | same | same |
| Inherited architect | worse | same | same | same | same |
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Junior A | 7 | 5.5 | 6 | 6 | 6 |
| Mid A | 7 | 5.5 | 6.5 | 6 | 6 |
| Maintainability senior | 7 | 6 | 6.5 | 6.5 | 6.5 |
| Inherited architect | 6.5 | 5.5 | 5.5 | 6.5 | 6 |
| Mean | 6.88 | 5.63 | 6.13 | 6.25 | 6.13 |
Mid A, decided by:
- Locality: `fejkdata.go:66` `WithDataFS`: every module is named at the call site, where the base silently started each generator from a package-level embed
- Navigation: `data/geose/fs.go:1` `geose`: a dotfile manifest decides the namespace and the folder follows a lowered, no-separator rule, one translation step from a failing path to its file
- Navigation: `internal/datafiles/datafiles.go:57` `Source.labelled`: a shipped file's error prints sv_SE (address.json), where the base printed the openable data/sv_SE/address.json
- Locality: `data.go:27` `addSource`: where a category lands depends on a field in a hidden file, a partial offset
Inherited architect, decided by:
- Navigation: `data/svse/fs.go:1` `svse`: the step from a dot path to a folder goes through the lowered-namespace rule, stated only in the README and each package doc
- Navigation: `internal/datafiles/datafiles.go:57` `labelled`: errors name sv_SE (address.json), not a path that opens as written
- Locality: `fejkdata.go:99` `New`: New loads exactly its options, cancelled by `data/svse/fs.go:1`'s dependency on geose that nothing in Go expresses
## 2026-10-11T13:08:34Z, PR #203 at fb2c809, against bf1becb
Ruling: better
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | same | better | same | same | better |
| Inherited architect | better | better | same | better | better |
Mid A, decided by:
- Locality, Overall: `fejkdata.go:66` `WithDataFS`: the caller names every module and New refuses no data naming the fix, where the base prepended a package-level shippedSource behind config{shipped: true}
- Locality: `data/sv_SE/fs.go:19` `Modules`: sv_SE reading geo.SE is a Go import with a doc line, not only a template reference inside address.json
- Shape: `internal/datafiles/mount.go:13` `mounted`: a hand-built virtual fs.FS replaces a base string, balanced by Source losing a field and Walk, Load and Manifest turning path-agnostic
Inherited architect, decided by:
- Locality, Navigation, Overall: `data/sv_SE/fs.go:19` `Modules`: a locale reading its country's places is a Go import, one hop in code where the base needed the README or a trace through JSON references
- Locality, Self-sufficiency: `fejkdata.go:66` `WithDataFS`: every source is named at the call site, and leaving them all out gives an error that spells the fix
- Navigation: `data/modules.go:17` `Modules`: shipped data has a Go home inside data/, each module's manifest beside its files
- Shape: `internal/datafiles/mount.go:13` `mounted`: seven small types fake the folders above the mount, balanced by Source keeping one form and truthful per-module packages
- Self-sufficiency: `data/sv_SE/fs.go:1` `svse`: each module says in code which namespace it supplies and what it reads
## 2026-10-11T13:18:20Z, PR #203 at 3d4131b, against bf1becb
Ruling: worse
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | same | better | worse | same | same |
| Inherited architect | same | worse | worse | same | worse |
| Maintainability senior | same | worse | same | same | worse |
| Junior A | same | worse | worse | better | worse |
Mid A, decided by:
- Shape: `internal/datafiles/mount.go:13` `mounted`: a hand-built virtual fs.FS of five types replaces a base string joined in dirPath
- Locality, Shape: `internal/datafiles/datafiles.go:34` `Source.Mounted`: one field drives file reads, error labels and the manifest read, and a second Mounted call does nothing
- Locality: `data/sv_SE/fs.go:19` `Modules`: a locale reading its country's places is a Go import with a package comment
- Locality: `fejkdata.go:99` `New`: every source is named at the call site, where the base loaded shipped data implicitly
- Shape: `data/modules.go:17` `Modules`: six packages, five near-identical boilerplate
Inherited architect, decided by:
- Locality, Shape, Overall: `internal/datafiles/mount.go:13` `mounted`: a fake fs.FS of five types with made-up folders above the mount, where the base had a base string; where it rots first
- Locality: `internal/datafiles/datafiles.go:34` `Source.Mounted`: the mount point kept in Source.at and mounted.at, and labelled undoes it through underMount
- Locality, Self-sufficiency: `index.go:22` `placeIndex`: index keys carry the full namespaced path, which only the mount maps back, stated nowhere at either site
- Locality: `fejkdata.go:41` `config.given`: every source explicit and errNoData names the fix, a partial offset
- Navigation: `data/modules.go:14` `go:generate`: the manifest generator lives in a test of the parent package
Maintainability senior, decided by:
- Locality, Overall: `internal/datafiles/mount.go:13` `mounted`: a fake fs.FS inventing the folders above the mount, so tracing why a file opens means following three paths in Open, where the base prefixed one base string
- Locality: `internal/datafiles/datafiles.go:56` `labelled`: one field serves the manifest read, file opens and error labels, and labelled undoes the mount through code in mount.go
- Locality: `fejkdata.go:99` `New`: the hidden shipped default is gone, a partial offset
Junior A, decided by:
- Shape, Locality, Overall: `internal/datafiles/mount.go:13` `mounted`: about 130 lines and five types fake folders so a module's root shows up under its namespace, where prefixing Category.Folders looks like the obvious alternative
- Locality: `internal/datafiles/datafiles.go:34` `Source.Mounted`: at changes both fsys and how errors label files, and a second Mounted call does nothing silently
- Self-sufficiency: `data/sv_SE/fs.go:1` `svse`: each package doc states its namespace and what it reads, and Modules names exactly what it pulls in
- Self-sufficiency: `fejkdata.go:79` `errNoData`: the error names the call that loads the shipped data
## 2026-10-11T18:50:05Z, PR #209 at fbe2fde, against d157e45
Ruling: better
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | same | better | same | same | better |
| Inherited architect | same | better | better | same | better |
Mid A, decided by:
- Locality, Overall: `cli/cli.go:318` `options`: the modules a run loads come in through Main and run, where the base fetched data.Modules() inside the flag handling
- Locality: `internal/datafiles/datafiles.go:56` `labelled`: reads Source.at, where the base type-asserted the FS to recover the mount point
- Locality, Shape: `internal/datafiles/mount.go:11` `mounted`: two types and one helper, no hidden read flag, where the base held five
- Locality: `internal/datafiles/datafiles.go:34` `Mounted`: the mount point is stored twice, and a second call is silently ignored
- Shape: `cli/cli.go:104` `no-shipped-data`: in a custom CLI the flag drops whatever Main was given, which only Main's doc says
- Shape, Self-sufficiency: `data-import/geo-us.py:219` `main`: the street share is a bare positional float, and places is rebound partway
Inherited architect, decided by:
- Shape, Locality: `internal/datafiles/mount.go:8` `mounted`: Open, ReadFile and ReadDir over one helper, where the base faked a tree in five types
- Locality: `internal/datafiles/manifest.go:53` `ManifestPath`: reads the explicit Source.at, though at is stored twice and only Mounted keeps them in step
- Locality: `cli/cli.go:318` `options`: the shipped set comes in through Main, run and options, where the base fixed it deep in the flag handling
- Shape: `cli/cli.go:104` `no-shipped-data`: once Main takes any modules, the flag's name promises less than it does
- Shape: `data-import/geo-us.py:211` `main`: places is rebound from every incorporated place to the shipped ones
- Locality: `cli/cli.go:651` `moduleVersion`: a pure function over the build info picks fejkdata's version by path
## 2026-10-11T19:02:27Z, PR #209 at 69af4c1, against d157e45
Ruling: better
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | same | better | better | same | better |
| Inherited architect | same | better | better | same | better |
Mid A, decided by:
- Shape, Locality: `internal/datafiles/mount.go:11` `mounted`: serves Open, ReadFile and ReadDir with one stateless entry type, where the base faked a full tree with a read flag
- Locality: `internal/datafiles/datafiles.go:18` `Source`: the mount point is a field, where labelled and ManifestPath type-asserted the FS; a second Mounted is silently ignored
- Locality: `cli/cli.go:321` `invocation.options`: the modules and the command name arrive as parameters, where the base called data.Modules() in the flag logic
- Shape: `cli/cli.go:678` `versionLine`: the CLI is its own package and main is one line, partly offset by the version line's branches and the COMMAND placeholder
Inherited architect, decided by:
- Shape, Locality: `internal/datafiles/mount.go:11` `mounted`: two types and one pure underMount, where the base faked an fs.File tree in five types with a read cursor
- Locality: `internal/datafiles/manifest.go:53` `ManifestPath`: the manifest reads through s.at whether called before or after Mounted, removing the base's order dependence
- Locality: `internal/datafiles/datafiles.go:34` `Mounted`: a second call is a silent no-op and at lives twice; where it rots first
- Shape: `cmd/fejkdata/main.go:12` `main`: the binary is one line over a library package, where the base bound the CLI to data.Modules() inside the flag handling
- Locality: `cli/cli.go:667` `report`: rewrites the library's prefix inside the joined error text, a string-level contract with another package
## 2026-10-12T16:29:54Z, PR #216 at 52e0304, against 90ffcff
Ruling: better
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | same | better | better | better | better |
| Inherited architect | same | better | same | same | better |
Mid A, decided by:
- Locality, Overall: `fejkdata.go:101` `New`: the seed is plain input, where the base drew it from crypto/rand through the swappable package-level randomBytes and drawSeed
- Self-sufficiency, Shape: `fejkdata.go:129` `config.misuse`: one named unit collects every option mistake, and errNoSeed tells the caller what to write
- Shape: `fejkdata.go:40` `config.dataGiven`: the name says what it tracks now that seeded sits beside it
- Self-sufficiency: `cli/cli.go:26` `usage`: --help states the --seed default and stays true for a custom CLI
- Self-sufficiency: `internal/jsonvalue/jsonvalue.go:29` `located`: a JSON syntax error names its line and column
- Shape: `record.go:216` `errChoiceRecord`: an inline errors.New became a named sentinel FakeRecord treats alike
- Locality: `Dockerfile:14` `release-tooling`: a /tested sentinel copy is an indirection, though its comment explains it
Inherited architect, decided by:
- Locality, Overall: `fejkdata.go:101` `New`: the seed is a required input and the package-level randomBytes global is gone; New reads straight through options, misuse, load, build
- Locality: `fejkdata.go:129` `misuse`: the base's two early returns become one unit joining every option mistake before any data loads
- Locality: `cli/cli.go:593` `run`: the CLI's fallback seed is a systemSeed parameter drawn once in Main, so everything that changes the output arrives through run's signature
## 2026-10-12T16:30:06Z, PR #216 at 52e0304
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Junior A | 7 | 5.5 | 6.5 | 6 | 6 |
| Mid A | 7 | 6 | 6.5 | 6.5 | 6.5 |
| Maintainability senior | 7 | 5.5 | 6.5 | 6.5 | 6 |
| Inherited architect | 7 | 6 | 6 | 7 | 6.5 |
| Mean | 7.00 | 5.75 | 6.38 | 6.50 | 6.25 |
Junior A, hardest first:
- `pick.go:179` `readName`: six pieces of state, with keptInPick's decision made at load through addressedKeys
- `path.go:190` `drawSteps`: climbed carries state between steps and pins swap mid-loop
- `table.go:253` `table.route`: six-field tableRoute plus fromRow and pinnedBelow held at once
- `node.go:65` `template`: filled in four phases by different files, with a change-together set
- `AGENTS.md:109` `misc.loglevel`: the decision title says keyed by keyword, the JSON keys by code
Mid A, hardest first:
- `pick.go:9` `pickKey`: keys built at load must match keys built at render, with heap-escape rules mixed in
- `render.go:146` `readField`: four read paths, and the name undersells it
- `path.go:190` `drawSteps`: order-dependent climbed and stepDownPins
- `datatype.go:230` `columnReadOf`: three spellings of a column read, with a four-function change-together contract
- `AGENTS.md:109` `misc.loglevel`: the decision title contradicts the JSON
Maintainability senior, hardest first:
- `render.go:146` `readField`: choosing named, kept or fresh needs env.pick, pickAt, under, levels and addressed at once
- `env.go:71` `enter`: frames pushed here must be found by frameOf later, order dependence across three files
- `path.go:190` `drawSteps`: climbed carries state from stepParent to the next stepChild
- `path.go:145` `compiledPath.walkEvery`: a capacity-clipping alias with no comment on why it is safe
- `index.go:109` `loadReached`: lazy loading mutates the shared tree and rolls back with putBack
Inherited architect, would restructure first:
- `pick.go:371` `keptInPick`: string pickKeys rebuilt at render must match addressedKeys built at load, and a miss silently draws afresh
- `node.go:253` `template`: one struct for six concepts, filled in four stages by four files
- `calc.go:219` `checkCalcFields`: calc lives in five places, and the root calc.go holds only load fences
- `datatype.go:222` `columnRead`: column-read resolution sits in datatype.go
- `cli/cli.go:505` `isPath`: copies the path grammar, the CLI's first rot point
## 2026-10-13T22:09:02Z, PR #220 at 909ddc5, against 891323d
Ruling: same
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | better | worse | same | same | better |
| Inherited architect | better | worse | same | same | same |
| Maintainability senior | better | worse | same | same | same |
| Junior A | better | worse | same | better | better |
Mid A, decided by:
- Navigation: `README.md:32` `table of contents` and `README.md:118` `### What stdin renders`: a link row reaches every top-level section, and stdin's byte rules sit under a heading of their own instead of an unheaded block between the CLI examples and the flag table.
- Navigation: `README.md:1171` `Layout` row for `release-tooling/` and `README.md:1277` `Development`: a packaging failure leads straight to `packages.yml` or `package-repos.Dockerfile`.
- Locality: `release-tooling/goreleaser.yaml:60` `isEnvSet "OLDER_TAG"` and `release-tooling/goreleaser.yaml:61` `dockerfile: ../tooling/release-tooling/image.Dockerfile`: the config assumes the workflow's src/ and tooling/ checkouts, and OLDER_TAG is set by a shell step in `packages.yml:72` with no note where it is read.
Inherited architect, decided by:
- Locality: `release-tooling/goreleaser.yaml:60` `dockers_v2` tags: whether the image gets `latest` hangs on an env var only `.github/workflows/packages.yml:72` `Channels holding one version` sets, and that step's `--skip` keeps the one-version rule in a second file by a second mechanism.
- Locality: `release-tooling/goreleaser.yaml:6` `dist: ../dist`: the config holds only under the two-checkout layout of `packages.yml:42`, and the `snapshot` stage of `package-repos.Dockerfile:5` works only because it skips docker.
- Locality: `.github/workflows/test.yml:35` `Relevant changes`: the packaging path list is restated in README.md:1222 and deferred to by AGENTS.md:3, and the copies already differ ("Go code" against `'*.go' go.mod go.sum`).
- Navigation: `README.md:32` table of contents, `README.md:36` `## Install` and `README.md:118` `### What stdin renders`: a symptom such as a missing trailing newline now leads to a heading.
- Navigation: `README.md:1277` release paragraph: one paragraph traces the changelog heading, `publish_release.py`, the tag output and `packages.yml`, with the secrets and re-packaging.
Maintainability senior, decided by:
- Locality: `release-tooling/goreleaser.yaml:6` `dist` and `release-tooling/goreleaser.yaml:61` `dockers_v2.dockerfile`: `../dist` and `../tooling/...` resolve only under the two sibling checkouts `packages.yml:42` sets up, and the header explains `dist` but not `../tooling/`.
- Locality: `.github/workflows/packages.yml:72` `Channels holding one version` and `release-tooling/goreleaser.yaml:60` `latest` tag template: OLDER_TAG is set in one file and read through `isEnvSet` in another, and the same step handles an older tag a second way through `--skip`.
- Locality: `Dockerfile:18` `packaging`, `release-tooling/package-repos.Dockerfile:5` `snapshot` and `.github/workflows/packages.yml:82` `goreleaser-action`: one GoReleaser version pinned in three files, and `packages.yml:56` reads the Go version by `sed` over the Dockerfile's ARG line.
- Locality: `.github/workflows/test.yml:35` `Relevant changes`: the packaging path list is written here and again as prose in README.md:1222, which AGENTS.md:3 points the gate at.
- Navigation: `README.md:32` table of contents and `README.md:118` `### What stdin renders`: the stdin byte rules have a heading to jump to.
- Navigation: `README.md:1171` `## Layout` and `README.md:1277` `## Development`: a cold reader goes from a wrong apt version to `packages.yml`, `package-repos.Dockerfile` and `check-deb`.
Junior A, decided by:
- Navigation: `README.md:32` table of contents and `README.md:36` `## Install`: a reader with a symptom jumps to a named section, and the stdin byte rules sit under `README.md:118` `### What stdin renders`.
- Navigation, Self-sufficiency: `README.md:1171` `release-tooling/` in Layout and `README.md:1222` `## Development`: a failed install or Packages run leads to `release-tooling/package-repos.Dockerfile`, and each secret is named.
- Locality: `.github/workflows/packages.yml:72` `Channels holding one version` and `release-tooling/goreleaser.yaml:60` `isEnvSet "OLDER_TAG"`: whether the image gets `latest` is decided by a variable set in another file, with no note where it is read.
- Locality: `Dockerfile:18` `packaging`, `release-tooling/package-repos.Dockerfile:5` `snapshot` and `.github/workflows/packages.yml:82` `version`: one GoReleaser version pinned in three places, tied only by renovate.json.
- Self-sufficiency: `release-tooling/goreleaser.yaml:1`, `.github/workflows/packages.yml:48` and `release-tooling/package-repos.Dockerfile:1`: each non-obvious choice is explained where it is made.
- Self-sufficiency: `doc.go:5` package doc: the namespace sentence now has its subject.
## 2026-10-14T09:22:16Z, PR #222 at e492511, against c3cc251
Ruling: same
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | same | same | same | same | same |
| Inherited architect | same | same | same | same | same |
## 2026-10-14T11:34:34Z, PR #225 at f734c9e, against f90b4d6
Ruling: worse
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|---|---|---|---|---|---|
| Mid A | same | worse | same | same | same |
| Inherited architect | same | worse | same | same | worse |
| Maintainability senior | same | worse | same | worse | worse |
| Junior A | same | worse | same | worse | worse |
Mid A, decided by:
- Locality: `Dockerfile:26` `vulnerability-filing`: the filer's test reads the real todo.md inside docker build ., so a plan edit can turn the Go gate red
- Locality: `vulnerability-filing/file_vulnerabilities.py:24` `ROW`: the filer splices todo.md as text, assuming its row shape, the Next ID marker and the Details heading
- Locality: `.github/workflows/test.yml:106` `release`: a release now also waits on vuln.go.dev's answer, visible only in a needs list
Inherited architect, decided by:
- Locality, Overall: `Dockerfile:26` `vulnerability-filing`: the plan is copied into the merge-gate build, so a planning edit that changes the table's shape fails docker build .
- Locality: `vulnerability-filing/file_vulnerabilities.py:101` `file_items`: edits todo.md as text; a missing Next ID crashes with exit 1, which reads like a vulnerability already planned
- Locality: `.github/workflows/test.yml:69` `vulnerabilities`: whether a release happens depends on vuln.go.dev that day, and every PR shows a red job while one is known
Maintainability senior, decided by:
- Locality: `vulnerability-filing/file_vulnerabilities.py:101` `file_items`: finds the release by an unnamed regex group, never checks the Next ID match, and splices Details by a replace that does nothing when the heading is missing
- Locality, Self-sufficiency: `.github/workflows/test.yml:83` `File each one in todo.md`: one exit code holds the release and triggers the proposal, which must tell filed, planned and crashed apart by a git diff
- Locality: `Dockerfile:26` `vulnerability-filing`: copies todo.md into the gate build, and the path filter lists todo.md
- Self-sufficiency: `README.md:1213` "Development": says the build runs the release tooling's tests and leaves out the filer's
- Self-sufficiency: `vulnerability-filing/file_vulnerabilities.py:2` module docstring: builds on rank_todo.py, which the repository does not hold
Junior A, decided by:
- Locality, Overall: `Dockerfile:26` `vulnerability-filing`: the Go merge gate copies todo.md and files an item into it, so editing the plan can turn the build red
- Locality: `.github/workflows/test.yml:106` `release`: whether a push to main releases also depends on vuln.go.dev's answer that day
- Self-sufficiency: `vulnerability-filing/file_vulnerabilities.py:24` `ROW`: the regex and the Details splice encode todo.md's layout, which the script never shows
- Self-sufficiency: `README.md:1213` "Development": the gate paragraph leaves out the filer's tests while AGENTS.md now defers to it
+1012
View File
File diff suppressed because it is too large Load Diff
+637
View File
@@ -0,0 +1,637 @@
# Reading cost
## Reading cost, chunk aedb337e0fa8, finished 2026-10-04T21:47:53Z
68314 new input tokens before the first commit.
Read most (opens and searches):
- `README.md` 9
- `todo.md` 6
- `compose.yaml` 3
- `data-import/territory.py` 2
- `data_test.go` 2
- `shipped_data_test.go` 2
- `AGENTS.md` 1
- `CHANGELOG.md` 1
- `data-import/tsv.py` 1
- `data/en_US/phone.json` 1
## Reading cost, chunk 501b569f8d46, finished 2026-10-04T22:23:51Z
118930 new input tokens before the first commit.
Read most (opens and searches):
- `docs/decisions.md` 8
- `README.md` 6
- `table_test.go` 4
- `render.go` 3
- `todo.md` 3
- `CHANGELOG.md` 2
- `pick.go` 2
- `table.go` 2
- `AGENTS.md` 1
- `fejkdata.go` 1
## Reading cost, chunk 2dd5dac8fb7d, finished 2026-10-04T22:50:50Z
93128 new input tokens before the first commit.
Read most (opens and searches):
- `README.md` 6
- `data_test.go` 4
- `todo.md` 3
- `format_stability_test.go` 2
- `CHANGELOG.md` 1
- `compose.yaml` 1
- `data/en_US/email.json` 1
- `data/en_US/first-name.tsv` 1
- `data/en_US/sex.tsv` 1
- `data/en_US/title.tsv` 1
## Reading cost, chunk 305e76235620, finished 2026-10-05T06:43:56Z
79628 new input tokens before the first commit.
Read most (opens and searches):
- `README.md` 5
- `todo.md` 5
- `format.go` 4
- `valueproof.go` 4
- `calc_test.go` 3
- `datatype_test.go` 2
- `name.go` 2
- `name_test.go` 2
- `record_test.go` 2
- `CHANGELOG.md` 1
## Reading cost, chunk 5d443718e378, finished 2026-10-05T08:24:10Z
366332 new input tokens before the first commit.
Read most (opens and searches):
- `table_test.go` 17
- `table.go` 9
- `struct.go` 8
- `README.md` 7
- `docs/decisions.md` 7
- `record.go` 5
- `calc.go` 4
- `data_test.go` 4
- `reference.go` 4
- `render.go` 4
## Reading cost, chunk 95fbdb646b25, finished 2026-10-05T09:41:42Z
705270 new input tokens before the first commit.
Read most (opens and searches):
- `docs/decisions.md` 16
- `todo.md` 16
- `README.md` 10
- `render.go` 5
- `table.go` 5
- `format.go` 4
- `AGENTS.md` 3
- `calc.go` 3
- `data.go` 3
- `fejkdata.go` 3
## Reading cost, chunk 790a992c1467, finished 2026-10-05T10:04:02Z
59846 new input tokens before the first commit.
Read most (opens and searches):
- `todo.md` 6
- `CHANGELOG.md` 2
- `README.md` 2
- `docs/decisions.md` 2
- `fejkdata.go` 2
- `fejkdata_test.go` 2
- `go.mod` 2
- `.dockerignore` 1
- `builtins_test.go` 1
- `cmd/fejkdata/main.go` 1
## Reading cost, chunk 2ef0d4af8d44, finished 2026-10-05T13:19:29Z
98933 new input tokens before the first commit.
Read most (opens and searches):
- `format_test.go` 5
- `todo.md` 5
- `calc.go` 4
- `calc_test.go` 4
- `format.go` 4
- `path.go` 3
- `reference.go` 3
- `README.md` 2
- `builtins_test.go` 2
- `inline.go` 2
## Reading cost, chunk ebcf577f6435, finished 2026-10-05T19:01:32Z
129910 new input tokens before the first commit.
Read most (opens and searches):
- `builtins_test.go` 7
- `todo.md` 5
- `format.go` 4
- `calc_test.go` 2
- `docs/decisions.md` 2
- `fejkdata.go` 2
- `reference.go` 2
- `CHANGELOG.md` 1
- `README.md` 1
- `calc.go` 1
## Reading cost, chunk 4f0407382a32, finished 2026-10-05T19:22:15Z
91864 new input tokens before the first commit.
Read most (opens and searches):
- `README.md` 4
- `table_test.go` 4
- `data.go` 3
- `todo.md` 3
- `graph.go` 2
- `node.go` 2
- `CHANGELOG.md` 1
- `compose.yaml` 1
- `fejkdata.go` 1
- `inline.go` 1
## Reading cost, chunk f695f44a8702, finished 2026-10-05T20:10:33Z
98023 new input tokens before the first commit.
Read most (opens and searches):
- `todo.md` 6
- `path_test.go` 4
- `README.md` 3
- `docs/decisions.md` 3
- `imports_test.go` 2
- `path.go` 2
- `reference_test.go` 2
- `table.go` 2
- `AGENTS.md` 1
- `CHANGELOG.md` 1
## Reading cost, chunk 958a19b8d709, finished 2026-10-05T20:49:27Z
115125 new input tokens before the first commit.
Read most (opens and searches):
- `name_test.go` 5
- `todo.md` 5
- `shape_test.go` 3
- `README.md` 2
- `graph.go` 2
- `inline.go` 2
- `node.go` 2
- `record_test.go` 2
- `struct.go` 2
- `CHANGELOG.md` 1
## Reading cost, chunk 35ed2f5f3f4c, finished 2026-10-05T21:03:20Z
53085 new input tokens before the first commit.
Read most (opens and searches):
- `AGENTS.md` 3
- `README.md` 3
- `todo.md` 3
- `.gitignore` 1
- `CHANGELOG.md` 1
## Reading cost, chunk ee081b4f8c13, finished 2026-10-05T21:20:27Z
67696 new input tokens before the first commit.
Read most (opens and searches):
- `todo.md` 3
- `pick_test.go` 2
- `render.go` 2
- `table_test.go` 2
- `CHANGELOG.md` 1
- `README.md` 1
- `data/geo/SE/address.json` 1
- `data/sv_SE/address.json` 1
- `docs/decisions.md` 1
- `pick.go` 1
## Reading cost, chunk 4c8a3bfa0fc7, finished 2026-10-06T22:46:06Z
75959 new input tokens before the first commit.
Read most (opens and searches):
- `todo.md` 5
- `README.md` 2
- `inline.go` 2
- `namefence.go` 2
- `path.go` 2
- `pick_test.go` 2
- `record.go` 2
- `AGENTS.md` 1
- `CHANGELOG.md` 1
- `compose.yaml` 1
## Reading cost, chunk 3fe488db154a, finished 2026-10-06T23:05:09Z
129293 new input tokens before the first commit.
Read most (opens and searches):
- `todo.md` 9
- `README.md` 6
- `format.go` 6
- `docs/decisions.md` 5
- `record.go` 5
- `inline.go` 4
- `render.go` 4
- `shipped.go` 4
- `CHANGELOG.md` 3
- `data.go` 3
## Reading cost, chunk 9dbdacef756b, finished 2026-10-06T23:24:42Z
79101 new input tokens before the first commit.
Read most (opens and searches):
- `README.md` 5
- `format.go` 5
- `todo.md` 5
- `path.go` 3
- `docs/decisions.md` 2
- `env.go` 2
- `pick.go` 2
- `table.go` 2
- `AGENTS.md` 1
- `CHANGELOG.md` 1
## Reading cost, chunk 8d30d77fba26, finished 2026-10-06T23:45:50Z
2986167 new input tokens before the first commit.
Read most (opens and searches):
- `README.md` 24
- `AGENTS.md` 18
- `CHANGELOG.md` 18
- `data-import/geo-se.py` 17
- `record.go` 17
- `cmd/fejkdata/main.go` 16
- `struct.go` 16
- `testdata/shipped_shape.txt` 16
- `.github/workflows/test.yml` 15
- `arm.go` 15
## Reading cost, chunk 0e8a426874fc, finished 2026-10-07T00:04:56Z
96842 new input tokens before the first commit.
Read most (opens and searches):
- `README.md` 6
- `CHANGELOG.md` 4
- `todo.md` 4
- `data-import/geo-se.py` 2
- `doc.go` 2
- `docs/decisions.md` 2
- `path.go` 2
- `resolve.go` 2
- `AGENTS.md` 1
- `arm.go` 1
## Reading cost, chunk 2b3baad177ba, finished 2026-10-07T00:27:52Z
152553 new input tokens before the first commit.
Read most (opens and searches):
- `README.md` 9
- `todo.md` 8
- `data-import/geo-us.py` 5
- `AGENTS.md` 4
- `fejkdata.go` 4
- `graph.go` 4
- `docs/decisions.md` 3
- `name.go` 3
- `pick.go` 3
- `resolve.go` 3
## Reading cost, chunk 4e18ae82f3dc, finished 2026-10-09T08:51:06Z
2852445 new input tokens before the first commit.
Read most (opens and searches):
- `README.md` 23
- `cmd/fejkdata/main.go` 23
- `pick.go` 18
- `AGENTS.md` 17
- `CHANGELOG.md` 16
- `compose.yaml` 16
- `env.go` 16
- `arm.go` 15
- `fejkdata.go` 15
- `path.go` 15
## Reading cost, chunk 99b75bd10e88, finished 2026-10-09T09:49:13Z
93957 new input tokens before the first commit.
Read most (opens and searches):
- `todo.md` 9
- `name.go` 6
- `README.md` 5
- `graph.go` 4
- `env.go` 3
- `path.go` 3
- `resolve.go` 3
- `arm.go` 2
- `compose.yaml` 2
- `docs/decisions.md` 2
## Reading cost, chunk 379810cdbae5, finished 2026-10-09T10:01:14Z
68500 new input tokens before the first commit.
Read most (opens and searches):
- `render_test.go` 6
- `todo.md` 3
- `README.md` 2
- `docs/decisions.md` 2
- `AGENTS.md` 1
- `CHANGELOG.md` 1
- `doc.go` 1
- `doc_test.go` 1
- `node.go` 1
- `render.go` 1
## Reading cost, chunk 3374028f1e29, finished 2026-10-09T10:35:44Z
69756 new input tokens before the first commit.
Read most (opens and searches):
- `todo.md` 6
- `README.md` 3
- `docs/decisions.md` 3
- `CHANGELOG.md` 2
- `AGENTS.md` 1
- `compose.yaml` 1
- `data-import/geo-se.py` 1
- `data-import/geo-us.py` 1
- `data-import/geo.py` 1
- `data-import/tsv.py` 1
## Reading cost, chunk c484375b01f5, finished 2026-10-09T10:53:55Z
60588 new input tokens before the first commit.
Read most (opens and searches):
- `todo.md` 6
- `README.md` 2
- `docs/decisions.md` 2
- `record.go` 2
- `render.go` 2
- `CHANGELOG.md` 1
- `docs/comprehension-history.md` 1
- `inline.go` 1
- `reference.go` 1
- `shipped.go` 1
## Reading cost, chunk 21603e0f3240, finished 2026-10-09T12:30:43Z
2855972 new input tokens before the first commit.
Read most (opens and searches):
- `README.md` 25
- `CHANGELOG.md` 22
- `cmd/fejkdata/main.go` 20
- `node.go` 19
- `testdata/shipped_shape.txt` 19
- `AGENTS.md` 18
- `arm.go` 17
- `DATA-LICENSES.md` 16
- `compose.yaml` 16
- `env.go` 16
## Reading cost, chunk f73b4c751678, finished 2026-10-09T13:02:23Z
340530 new input tokens before the first commit.
Read most (opens and searches):
- `todo.md` 15
- `docs/decisions.md` 14
- `README.md` 4
- `.github/workflows/test.yml` 2
- `compose.yaml` 2
- `fejkdata.go` 2
- `go.mod` 2
- `release-tooling/publish_release.py` 2
- `shipped.go` 2
- `shippedindex.go` 2
## Reading cost, chunk af1b90b1481c, finished 2026-10-09T17:00:45Z
91452 new input tokens before the first commit.
Read most (opens and searches):
- `README.md` 7
- `todo.md` 5
- `cmd/fejkdata/main_test.go` 3
- `CHANGELOG.md` 2
- `docs/decisions.md` 2
- `imports_test.go` 2
- `.dockerignore` 1
- `.github/workflows/test.yml` 1
- `cmd/fejkdata/main.go` 1
- `compose.yaml` 1
## Reading cost, chunk 2f68b5245844, finished 2026-10-10T10:55:47Z
112728 new input tokens before the first commit.
Read most (opens and searches):
- `README.md` 9
- `todo.md` 6
- `docs/decisions.md` 5
- `shipped_test.go` 2
- `CHANGELOG.md` 1
- `compose.yaml` 1
- `data.go` 1
- `fejkdata.go` 1
- `inline.go` 1
- `internal/datafiles/datafiles.go` 1
## Reading cost, chunk 1c769e839e3a, finished 2026-10-11T14:05:12Z
200600 new input tokens before the first commit.
Read most (opens and searches):
- `README.md` 13
- `todo.md` 8
- `CHANGELOG.md` 6
- `cmd/fejkdata/main.go` 6
- `docs/decisions.md` 6
- `shipped_test.go` 6
- `DATA-LICENSES.md` 5
- `bench_test.go` 5
- `data_test.go` 5
- `data.go` 4
## Reading cost, chunk 8ce3be08e54e, finished 2026-10-11T16:36:06Z
87725 new input tokens before the first commit.
Read most (opens and searches):
- `todo.md` 6
- `README.md` 4
- `data_test.go` 4
- `CHANGELOG.md` 2
- `docs/decisions.md` 2
- `compose.yaml` 1
- `data-import/geo-se.py` 1
- `data-import/geo-us.py` 1
- `data-import/geo.py` 1
- `data-import/source.py` 1
## Reading cost, chunk a8085e6ee71e, finished 2026-10-11T17:14:17Z
90566 new input tokens before the first commit.
Read most (opens and searches):
- `data-import/geo-us.py` 6
- `README.md` 4
- `todo.md` 4
- `data-import/geo-se.py` 3
- `compose.yaml` 2
- `data/geo/US/locality.tsv` 2
- `data_test.go` 2
- `.gitignore` 1
- `AGENTS.md` 1
- `CHANGELOG.md` 1
## Reading cost, chunk c1a0ce6b3d32, finished 2026-10-11T18:41:31Z
76079 new input tokens before the first commit.
Read most (opens and searches):
- `todo.md` 10
- `docs/decisions.md` 6
- `README.md` 4
- `AGENTS.md` 2
- `data-import/geo-se.py` 2
- `CHANGELOG.md` 1
- `data-import/geo-us.py` 1
- `data-import/geo.py` 1
## Reading cost, chunk 0cd4f425fa6f, finished 2026-10-11T21:08:07Z
102267 new input tokens before the first commit.
Read most (opens and searches):
- `README.md` 7
- `CHANGELOG.md` 6
- `todo.md` 6
- `cli/cli_test.go` 4
- `docs/decisions.md` 3
- `cmd/fejkdata/main_test.go` 2
- `AGENTS.md` 1
- `cmd/fejkdata/main.go` 1
- `compose.yaml` 1
- `data/modules.go` 1
## Reading cost, chunk d67229c6358e, finished 2026-10-11T21:39:58Z
372546 new input tokens before the first commit.
Read most (opens and searches):
- `README.md` 30
- `todo.md` 10
- `cli/cli.go` 5
- `cli/cli_test.go` 3
- `CHANGELOG.md` 2
- `data/sv_SE/.fejkdata.json` 2
- `docs/decisions.md` 2
- `internal/datafiles/datafiles.go` 2
- `record.go` 2
- `compose.yaml` 1
## Reading cost, chunk aca06ea931a8, finished 2026-10-12T15:34:32Z
78327 new input tokens before the first commit.
Read most (opens and searches):
- `todo.md` 8
- `README.md` 5
- `CHANGELOG.md` 1
- `DATA-LICENSES.md` 1
- `data/misc/car.tsv` 1
- `docs/decisions.md` 1
- `docs/research/research-sources-se.md` 1
- `docs/research/research-sources-world.md` 1
## Reading cost, chunk 2f1148286c4c, finished 2026-10-12T15:50:50Z
60817 new input tokens before the first commit.
Read most (opens and searches):
- `README.md` 4
- `todo.md` 3
- `CHANGELOG.md` 2
- `shipped_data_test.go` 2
- `.dockerignore` 1
- `.gitignore` 1
- `data_test.go` 1
- `docs/decisions.md` 1
- `release-tooling/publish_release.py` 1
- `shipped_test.go` 1
## Reading cost, chunk f26a17e98714, finished 2026-10-12T16:07:59Z
78241 new input tokens before the first commit.
Read most (opens and searches):
- `README.md` 10
- `todo.md` 7
- `CHANGELOG.md` 3
- `docs/decisions.md` 3
- `compose.yaml` 1
- `data/en_US/itin.json` 1
- `data/misc/timezone.json` 1
- `data/misc/timezone.tsv` 1
- `go.mod` 1
+10
View File
@@ -0,0 +1,10 @@
# Population by age band — research 2026-10-04
Share of the population in four bands, from each country's statistics office, for weighting a drawn birthdate by age. Computed from single-year ages.
| Country | Date | Total | 0–17 | 18–64 | 65–99 | 100+ | Source |
|---|---|---|---|---|---|---|---|
| SE | 2024-12-31 | 10,587,710 | 20.36% | 58.80% | 20.81% | 0.028% (2,961) | SCB Statistikdatabasen, table BE0101A/BefolkningNy, CC0 1.0 |
| US | 2024-07-01 | 340,110,988 | 21.50% | 60.51% | 17.96% | 0.029% (98,034) | Census Bureau, [nc-est2024-agesex-res.csv](https://www2.census.gov/programs-surveys/popest/datasets/2020-2024/national/asrh/nc-est2024-agesex-res.csv), public domain |
At 0.03%, a fixture of 10,000 rows holds about three people aged 100 or more.
+3 -2
View File
@@ -1,7 +1,7 @@
# Swedish (sv_SE) fake-data sources — research 2026-09-17 # Swedish (sv_SE) fake-data sources — research 2026-09-17
Scope: names, identifiers, phone, company, plates/cars, words, dates/misc, other. Geography excluded. Scope: names, identifiers, phone, company, plates/cars, words, dates/misc, other. Geography excluded.
"Verified" = fetched and read this session. Local copies of inspected files: `/home/lilleman/.claude/jobs/e377c8d4/tmp/`. "Verified" = fetched and read for this research.
## Licence summary (embed in MIT repo?) ## Licence summary (embed in MIT repo?)
@@ -15,7 +15,7 @@ Scope: names, identifiers, phone, company, plates/cars, words, dates/misc, other
| Bolagsverket/SCB bulk company files | Not stated ("värdefulla datamängder", free, no agreement) | Derive stats only; do not embed raw | Cite | | Bolagsverket/SCB bulk company files | Not stated ("värdefulla datamängder", free, no agreement) | Derive stats only; do not embed raw | Cite |
| Bankinfrastruktur clearing CSV | Not stated; accuracy disclaimer | Facts only; flag | Cite BSAB | | Bankinfrastruktur clearing CSV | Not stated; accuracy disclaimer | Facts only; flag | Cite BSAB |
| Bankgirot PDFs | "© Bankgirocentralen BGC AB. All rights reserved." (Informationsklass: Öppen) | No verbatim copy; facts only | — | | Bankgirot PDFs | "© Bankgirocentralen BGC AB. All rights reserved." (Informationsklass: Öppen) | No verbatim copy; facts only | — |
| PTS numbering plan PDF | Not stated (no licence on pts.se or in PDF) | Facts only; flag | Cite PTS | | PTS numbering plan PDF, pts.se number pages | Not stated (no licence on pts.se or in PDF) | Facts only; flag | Cite PTS |
| Transportstyrelsen blocked combinations | Not stated on page | Facts (a list of 99 codes); flag | Cite | | Transportstyrelsen blocked combinations | Not stated on page | Facts (a list of 99 codes); flag | Cite |
| SALDO morphology (Språkbanken) | CC BY 4.0 | Yes | Cite: Språkbanken (2017) SALDO's morphology, DOI 10.23695/agcm-ny22 | | SALDO morphology (Språkbanken) | CC BY 4.0 | Yes | Cite: Språkbanken (2017) SALDO's morphology, DOI 10.23695/agcm-ny22 |
| Språkbanken word statistics | CC BY 4.0 | Yes | Cite Språkbanken | | Språkbanken word statistics | CC BY 4.0 | Yes | Cite Språkbanken |
@@ -90,6 +90,7 @@ Recommendation: embed SCB 2022 xlsx-derived lists (top-N per sheet with counts)
## 3. Phone ## 3. Phone
- Source: PTS "The Swedish numbering plan for telephony according to ITU-T E.164", 2024-01-08 (verified, text extracted): https://pts.se/globalassets/globala-block/nummertillstand/the-swedish-numbering-plan-for-telephony-according-to-itu---2024-01-08.pdf. Swedish "nrplansammanstallning-2026-02-04.pdf" and https://nummer.pts.se/NbrPlanSearch are behind a Radware bot check (both curl and Playwright blocked). No licence stated by PTS anywhere found; pts.se says contact pts@pts.se for data. - Source: PTS "The Swedish numbering plan for telephony according to ITU-T E.164", 2024-01-08 (verified, text extracted): https://pts.se/globalassets/globala-block/nummertillstand/the-swedish-numbering-plan-for-telephony-according-to-itu---2024-01-08.pdf. Swedish "nrplansammanstallning-2026-02-04.pdf" and https://nummer.pts.se/NbrPlanSearch are behind a Radware bot check (both curl and Playwright blocked). No licence stated by PTS anywhere found; pts.se says contact pts@pts.se for data.
- Reserved for fiction, PTS "Telefonnummer till böcker och filmer" (verified 2026-09-21; curl and Playwright are Radware-blocked, a rendering fetcher gets through): https://www.pts.se/internet-och-telefoni/telefonnummer-och-adressering/telefonnummer-till-bocker-och-filmer/ — mobile 070-1740605–1740699, geographic 08-46500400–46500499 (Stockholm), 031-3900600–3900699 (Göteborg), 040-6280400–6280499 (Malmö), 0980-319200–319299 (Kiruna). 495 numbers PTS holds back so they reach nobody, which is goal 15's never-reaching path for `sv_SE.phone`. Same licence status as the numbering plan: none stated, facts only, cite PTS. The mobile block fits the shipped `{a} {b} {c}` grouping (`174 06 05`–`99`), as do 031 and 040; Stockholm's 8 subscriber digits and Kiruna's 6 need a format of their own, and 0980 is not among the 20 shipped area codes.
- 264 geographic area codes (PDF text yields 263 "Area code for" rows; one likely split across a page break — flag). Area code = trunk "0" + NDC of 1–3 digits (08 Stockholm, 031 Göteborg, 040 Malmö, 0480 Kalmar). Geographic N(S)N (NDC + subscriber) max 9 digits, min 7 (2-digit NDC) or 8 (3-digit NDC); Stockholm min 7. Each row in the PDF gives NDC, max/min length, "Area code for <name>", so a `{ndc, name}` list can be built from it. - 264 geographic area codes (PDF text yields 263 "Area code for" rows; one likely split across a page break — flag). Area code = trunk "0" + NDC of 1–3 digits (08 Stockholm, 031 Göteborg, 040 Malmö, 0480 Kalmar). Geographic N(S)N (NDC + subscriber) max 9 digits, min 7 (2-digit NDC) or 8 (3-digit NDC); Stockholm min 7. Each row in the PDF gives NDC, max/min length, "Area code for <name>", so a `{ndc, name}` list can be built from it.
- Mobile: NDC 70, 72, 73, 76, 79 — N(S)N exactly 9 digits, i.e. 07X-XXX XX XX (10 digits with trunk 0). Others: 71 mobile broadband/M2M (13 digits), 74 paging, 75 personal numbering, 77 shared cost, 20 freephone, 10 location-independent (10 AXX XX XX, A=1–8), 378 M2M fixed (10 digits). - Mobile: NDC 70, 72, 73, 76, 79 — N(S)N exactly 9 digits, i.e. 07X-XXX XX XX (10 digits with trunk 0). Others: 71 mobile broadband/M2M (13 digits), 74 paging, 75 personal numbering, 77 shared cost, 20 freephone, 10 location-independent (10 AXX XX XX, A=1–8), 378 M2M fixed (10 digits).
- Formatting (Språkrådet/Isof frågelådan, via search excerpts): hyphen after area code, subscriber digits grouped 2–3 from the left: `08-668 01 50`, `031-123 45 67`, `0480-123 45`, `070-123 45 67` (also `0701-23 45 67` accepted). International: `+46 70 123 45 67` (drop trunk 0). - Formatting (Språkrådet/Isof frågelådan, via search excerpts): hyphen after area code, subscriber digits grouped 2–3 from the left: `08-668 01 50`, `031-123 45 67`, `0480-123 45`, `070-123 45 67` (also `0701-23 45 67` accepted). International: `+46 70 123 45 67` (drop trunk 0).
-461
View File
@@ -1,461 +0,0 @@
package fejkdata
import (
"fmt"
"sort"
"strings"
)
// drawSet is one render's reference draws: the unnamed draw group's, and each named one's.
type drawSet struct {
unnamed draws
named map[string]*draws
}
// drawScope is where a render reads its reference paths: a draw set, in the draw group of the
// template rendering, and the row a table's format is rendering, which its columns read.
type drawScope struct {
set *drawSet
group string
t *table
row int
}
// newDrawSet makes the unnamed group's maps where the set is declared, keeping them on that frame's
// stack for a render that reads through them; a zero drawSet makes them on its first read instead.
func newDrawSet(s *session) drawSet {
return drawSet{unnamed: draws{variant: map[string]node{}, value: map[string]draw{}, s: s}}
}
// renderOnce renders n as one render, over draws of its own.
func renderOnce(s *session, n node) string {
set := drawSet{unnamed: draws{s: s}}
return render(s, n, drawScope{set: &set})
}
// in is the scope t renders in: its draw group where it names one, else its caller's.
func (sc drawScope) in(t *template) drawScope {
if t.drawGroupKey != "" {
sc.group = t.drawGroupKey
}
return sc
}
// draws is the set's draws for sc's draw group. The session is passed in rather than
// read out of the set: copying it from there would leak the set's maps to the heap.
func (sc drawScope) draws(s *session) *draws {
if sc.group == "" {
return &sc.set.unnamed
}
d, drew := sc.set.named[sc.group]
if !drew {
if sc.set.named == nil {
sc.set.named = map[string]*draws{}
}
d = &draws{variant: map[string]node{}, value: map[string]draw{}, s: s}
sc.set.named[strings.Clone(sc.group)] = d // a key from sc would leak sc, and with it every render's draw set, to the heap
}
return d
}
// drawGroupOf reads a template's "drawGroup" (default ""), which a repeat cannot carry: each
// iteration renders in no draw group.
func drawGroupOf(m map[string]any, repeat int) (string, error) {
v, ok := m["drawGroup"]
if !ok {
return "", nil
}
name, ok := v.(string)
switch {
case !ok:
return "", fmt.Errorf("drawGroup must be a string, got %T", v)
case name == "":
return "", fmt.Errorf(`drawGroup "" is the default, so it has no effect; drop it`)
case repeat > 1:
return "", fmt.Errorf("drawGroup %q on a repeat names nothing, since each iteration is a render of its own; drop it", name)
}
return name, nil
}
// keyDrawGroup keys t's draw group by the category t sits in, "" for an inline template, so a name
// is local to its category.
func (t *template) keyDrawGroup(category string) {
if t.drawGroup != "" {
t.drawGroupKey = category + "/" + t.drawGroup
}
}
// checkNestedDrawGroup refuses a template beneath one drawing in group that names group again,
// short of a repeat or another draw group.
func checkNestedDrawGroup(fields map[string]node, group string) error {
if group == "" {
return nil
}
var walk func(path string, n node) error
walk = func(path string, n node) error {
t, isTemplate := n.(*template)
switch {
case isTemplate && t.drawGroup == group:
return fmt.Errorf("%q names drawGroup %q, the draw group this template draws in already; drop it", path, group)
case isTemplate && (t.drawGroup != "" || t.repeat > 1):
return nil
}
for _, c := range contained(n) {
if err := walk(join(path, c.name), c.node); err != nil {
return err
}
}
return nil
}
for _, name := range sortedNames(fields) {
if err := walk(name, fields[name]); err != nil {
return err
}
}
return nil
}
// drawCheck fences each template of a scope as a render of its own, remembering which nodes read a
// reference path.
type drawCheck struct {
memo map[readsMemo]bool
}
type readsMemo struct {
n node
stopAtGroup bool
}
// checkDrawGroup refuses a draw group that splits nothing: one whose render reads every reference
// path inside a repeat or a nested draw group, which draw apart from it whatever it names.
func (c *drawCheck) checkDrawGroup(path string, n node) error {
if t, ok := n.(*template); ok && t.drawGroup != "" && !c.splitsDraws(t) {
return fmt.Errorf("%s: drawGroup %q splits nothing, since nothing it renders reads a reference path outside a repeat or a nested drawGroup; drop it", path, t.drawGroup)
}
return nil
}
func (c *drawCheck) checkDraws(path string, n node) error {
t, ok := n.(*template)
if !ok {
return nil
}
if err := checkOwnFamily(t); err != nil {
return fmt.Errorf("%s: %w", path, err)
}
if !c.readsPath(t) {
return nil
}
w := newDrawWalk()
for _, e := range renderEdges(t) {
w.edge(t, e, drawAt{group: t.drawGroupKey, route: drawRoute{e.reached(), e.label}})
}
if err := w.check(); err != nil {
return fmt.Errorf("%s: %w", path, err)
}
return nil
}
// checkOwnFamily refuses a table's format or cell that reads, however many templates
// away and through a repeat or a draw group too, a table of its own family: a row
// rendered whole draws its row without pinning it, so the family would draw apart
// from the row being rendered, whichever draws the reaching template holds.
func checkOwnFamily(t *template) error {
own := t.table
if own == nil {
own = t.cellOf
}
if own == nil {
return nil
}
seen := map[node]bool{}
var find func(n node) (renderEdge, *table, bool)
find = func(n node) (renderEdge, *table, bool) {
if seen[n] {
return renderEdge{}, nil, false
}
seen[n] = true
for _, e := range renderEdges(n) {
if a, isRef := refRead(n, e.label); isRef {
if head, isTable := n.(*template).fields[a.key].(*table); isTable && head.family() == own.family() {
return e, head, true
}
}
if e, head, found := find(e.to); found {
return e, head, true
}
}
return renderEdge{}, nil, false
}
if e, head, found := find(t); found {
return fmt.Errorf("%s reads %s, a table of its own family, which a row of %s rendered whole would draw apart from; read the family from a template beside it, or add the value as a column", e.reached(), head.category, own.category)
}
return nil
}
// checkRecordDraws fences the columns of a record — a template compiled at the top without a
// repeat — so a load proves the record view of it as well as the string view.
func (c *drawCheck) checkRecordDraws(path string, n node) error {
t, ok := n.(*template)
if !ok || !t.record {
return nil
}
// A record-only template's format renders nothing, so weigh the columns, not the format.
columns := recordColumns(t)
reads := false
for _, name := range columns {
reads = reads || c.readsPath(t.fields[name])
}
if !reads {
return nil
}
if err := checkColumnDraws(t, columns); err != nil {
return fmt.Errorf("%s: %w", path, err)
}
return nil
}
// readsPath reports whether rendering n reads a reference path, short of a repeat, which renders
// over draws of its own.
func (c *drawCheck) readsPath(n node) bool { return c.reads(n, false) }
// splitsDraws reports whether rendering n reads a reference path that n's own draw group answers
// for: one outside a repeat and outside a nested draw group, which hold their own draws.
func (c *drawCheck) splitsDraws(n node) bool { return c.reads(n, true) }
// reads walks what rendering n renders for a reference path, stopping at a repeat — and at a nested
// draw group when stopAtGroup — since each holds draws of its own.
func (c *drawCheck) reads(n node, stopAtGroup bool) bool {
k := readsMemo{n, stopAtGroup}
if r, done := c.memo[k]; done {
return r
}
r := false
for _, e := range renderEdges(n) {
if a, isRef := refRead(n, e.label); (isRef && (len(a.tail) > 0 || readsTable(n, a))) || (!repeats(e.to) && !(stopAtGroup && grouped(e.to)) && c.reads(e.to, stopAtGroup)) {
r = true
break
}
}
if c.memo == nil {
c.memo = map[readsMemo]bool{}
}
c.memo[k] = r
return r
}
func repeats(n node) bool {
t, isTemplate := n.(*template)
return isTemplate && t.repeat > 1
}
func grouped(n node) bool {
t, isTemplate := n.(*template)
return isTemplate && t.drawGroupKey != ""
}
// readsTable reports whether a reference of n names a table, whose draw the render's
// group answers for even when read whole.
func readsTable(n node, a arm) bool {
t, isTemplate := n.(*template)
if !isTemplate {
return false
}
_, isTable := t.fields[a.key].(*table)
return isTable
}
// refRead is the reference an edge of n reads; false when the edge reads none.
func refRead(n node, label string) (arm, bool) {
t, isTemplate := n.(*template)
if !isTemplate {
return arm{}, false
}
a := splitArm(label, t.refs)
return a, isRef(a.key)
}
// checkColumnDraws fences a record's columns as one render.
func checkColumnDraws(t *template, columns []string) error {
w := newDrawWalk()
for _, name := range columns {
w.walk(t.fields[name], drawAt{group: t.drawGroupKey, route: drawRoute{spelling: fmt.Sprintf("column %q", name)}})
}
return w.check()
}
// drawWalk gathers the references one render reads, by draw group, for check to compare.
type drawWalk struct {
reads []pathRead
read map[drawKey]bool
seen map[drawVisit]bool
}
// drawAt is where a walk stands: the draw group it draws in, how the render's root reached it, and
// the table rows it entered.
type drawAt struct {
group string
route drawRoute
alt rowSet
}
// rowSet is the rows a walk entered, one per table: only one row of a table renders, so reads in
// two rows of one table never meet, while reads in one row, across its columns and whatever they
// reach, do.
type rowSet []tablePin
func (s rowSet) rowOf(t *table) (int, bool) {
for _, p := range s {
if p.t == t {
return p.row, true
}
}
return 0, false
}
// enter is s with row r of t, where t is not in it yet; the set is copied, since walks branch.
func (s rowSet) enter(t *table, r int) rowSet {
if _, in := s.rowOf(t); in {
return s
}
out := append(append(make(rowSet, 0, len(s)+1), s...), tablePin{t, r})
sort.Slice(out, func(i, j int) bool { return out[i].t.category < out[j].t.category })
return out
}
// key spells the set for a map, by the tables' identities.
func (s rowSet) key() string {
var b strings.Builder
for _, p := range s {
fmt.Fprintf(&b, "%p[%d]", p.t, p.row)
}
return b.String()
}
// alternatives reports whether two reads sit in different rows of one table.
func alternatives(a, b drawAt) bool {
for _, p := range a.alt {
if r, in := b.alt.rowOf(p.t); in && r != p.row {
return true
}
}
return false
}
// drawRoute is how a render reaches a draw: as its author spells it, and the root edge's label.
type drawRoute struct{ spelling, label string }
// pathRead is one reference a render reads: a path, or a bare reference with no tail.
type pathRead struct {
at drawAt
a arm
tr *tableRead // set where the reference names a table
}
type drawVisit struct {
n node
group string
alt string
}
type drawKey struct {
group string
path string
alt string
}
func newDrawWalk() *drawWalk {
return &drawWalk{read: map[drawKey]bool{}, seen: map[drawVisit]bool{}}
}
// walk follows what rendering n renders. A repeat renders over draws of its own, so the walk stops
// there.
func (w *drawWalk) walk(n node, at drawAt) {
v := drawVisit{n, at.group, at.alt.key()}
if w.seen[v] {
return
}
w.seen[v] = true
if repeats(n) {
return
}
if t, isTemplate := n.(*template); isTemplate && t.drawGroupKey != "" {
at.group = t.drawGroupKey
}
for _, e := range renderEdges(n) {
if cell, isCell := e.to.(*template); isCell && cell.cellOf != nil {
if at.alt.excludes(cell) {
continue
}
w.edge(n, e, drawAt{at.group, at.route, at.alt.enter(cell.cellOf, cell.cellRow)})
continue
}
w.edge(n, e, at)
}
}
// excludes reports whether a cell's row cannot render with the rows entered: another
// row of its table, or a row outside an entered ancestor's.
func (s rowSet) excludes(cell *template) bool {
for _, p := range s {
if p.t == cell.cellOf {
return p.row != cell.cellRow
}
if cell.cellOf.descends(p.t) && !cell.cellOf.under(cell.cellRow, p.t, p.row) {
return true
}
}
return false
}
// edge records the reference an edge reads, then walks on with every row the read
// pins entered, so a selected row renders only its own cells.
func (w *drawWalk) edge(from node, e renderEdge, at drawAt) {
if a, reads := refRead(from, e.label); reads {
tr := tableReadOf(from.(*template).fields[a.key], a, e.to)
if k := (drawKey{at.group, a.path, at.alt.key()}); !w.read[k] {
w.read[k] = true
w.reads = append(w.reads, pathRead{at, a, tr})
}
if tr != nil {
tr.pins.each(func(t *table, r int) { at.alt = at.alt.enter(t, r) })
}
}
w.walk(e.to, at)
}
// check refuses what one draw per reference path cannot answer for: a read of a level beside a path
// another read takes into it, and two reads of one table family that select different rows. Reads
// are compared in path order, so which pair is reported does not vary.
func (w *drawWalk) check() error {
sort.SliceStable(w.reads, func(i, j int) bool {
if w.reads[i].at.group != w.reads[j].at.group {
return w.reads[i].at.group < w.reads[j].at.group
}
return w.reads[i].a.path < w.reads[j].a.path
})
for i, level := range w.reads {
for _, into := range w.reads[i+1:] {
if into.at.group != level.at.group || alternatives(level.at, into.at) {
continue
}
if strings.HasPrefix(into.a.path, level.a.path+".") && !(level.tr != nil && level.tr.whole) {
return overlapError(level.at.route, level.a.name, into)
}
}
}
return checkFamilies(w.reads)
}
func overlapError(route drawRoute, ref string, into pathRead) error {
return fmt.Errorf("%s renders a level that %s reads a path into; name the fields you want instead, or draw them apart with a drawGroup", route.spelled(ref), into.at.route.spelled(into.a.name))
}
// spelled names the route, and the reference it reaches a draw by where its root edge is not that
// reference.
func (r drawRoute) spelled(ref string) string {
if ref == "" || ref == r.label {
return r.spelling
}
return fmt.Sprintf("%s with {%s}", r.spelling, ref)
}
+113
View File
@@ -0,0 +1,113 @@
package fejkdata
import (
"github.com/larvit/fejkdata/internal/drawstate"
"github.com/larvit/fejkdata/internal/invariant"
)
// readValue is what one read drew: its text, and whether it landed on a null.
type readValue struct {
text string
null bool
}
// frameStack is the frames of the name scopes rendering, innermost last. A render's env points
// at it, and a repeat iteration's shares it: holding the frames in a renderEnv would move every
// render's env to the heap.
type frameStack struct {
frames []*pickFrame
}
func (st *frameStack) push(f *pickFrame) int {
mark := len(st.frames)
st.frames = append(st.frames, f)
return mark
}
func (st *frameStack) pop(mark int) { st.frames = st.frames[:mark] }
// renderEnv is where a render reads: the frames of the name scopes rendering; and row, the row
// of the table rendering, which its columns read. It passes by value, and what is read from it
// reaches a map key, an interface or a func value only as a copy; else env, and with it every
// render's frames, moves to the heap.
type renderEnv struct {
frames *frameStack
row renderedRow
// base is the depth of the frame stack when the read rendering the category entered it,
// below which no frame is the category's.
base int
// pick is the named pick being rendered, nil outside one; pickAt is the key in it of what renders.
pick *namedPick
pickAt pickKey
}
type renderedRow struct {
t *table
index int
}
// at returns the env n renders in, n being the leaf of a path whose rows sit in pins; where n
// is a table's row or column, the env carries the row pins holds for that table.
func (env renderEnv) at(n node, pins *pinSet) renderEnv {
switch n := n.(type) {
case *tableRow:
env.row = renderedRow{n.t, pins.MustRow(n.t.rows)}
case *tableColumn:
env.row = renderedRow{n.t, pins.MustRow(n.t.rows)}
}
return env
}
func (env renderEnv) rowOf(t *table) int {
if env.row.t != t {
panic(invariant.Broken("a column of %s renders in an env holding no row of it", t.rows.Segment()))
}
return env.row.index
}
// enter starts a read landing on n: the read sees no frame opened before it, and gets a frame for
// each name scope around n, from memo where there is one, so reads sharing memo read one pick of
// each name. It returns the mark that closes those frames.
func (env renderEnv) enter(n node, memo *drawMemo) (renderEnv, int) {
env = env.entering()
for scope := scopeAround(n); scope != nil; scope = scope.up {
if len(scope.order) > 0 {
env.frames.push(memo.enteredFrame(scope))
}
}
return env, env.base
}
// entering is env as a read entering a category sees it: no frame opened before it, and no pick.
func (env renderEnv) entering() renderEnv {
env.base, env.pick = len(env.frames.frames), nil
return env
}
// renderFrame opens a fresh frame of t's scope, where t binds names and no read entering the
// category opened one, returning the mark that closes it, or -1.
func (env renderEnv) renderFrame(t *template) int {
own := t.ownScope()
if own == nil || env.frameOf(own) != nil {
return -1
}
return env.frames.push(newPickFrame(own))
}
// frameOf is the frame of scope rendering since the read entering the category, nil where none is.
func (env renderEnv) frameOf(scope *nameScope) *pickFrame {
for i := len(env.frames.frames) - 1; i >= env.base; i-- {
if f := env.frames.frames[i]; f.scope == scope {
return f
}
}
return nil
}
// drawRowOf draws the row a render of t reads: inside the pick's rows where t renders as part of one.
func (env renderEnv) drawRowOf(s *drawstate.State, t *table) int {
if env.pick != nil {
return t.rows.DrawIn(s, &env.pick.pins)
}
return t.rows.Draw(s)
}
-319
View File
@@ -1,319 +0,0 @@
package fejkdata
import (
"fmt"
"sort"
"strings"
)
// tablePin is one table's pinned row.
type tablePin struct {
t *table
row int
}
// pinned is the row the render pinned for t, if any.
func (d *draws) pinned(t *table) (int, bool) {
for _, p := range d.pins[:d.npins] {
if p.t == t {
return p.row, true
}
}
r, ok := d.more[t]
return r, ok
}
// mustRow is the row pinned for t, which the walk reaching a column pinned.
func (d *draws) mustRow(t *table) int {
r, ok := d.pinned(t)
if !ok {
panic(fmt.Sprintf("fejkdata: a column of %s is rendered with no row pinned", t.category))
}
return r
}
// pin pins row r of t, and the rows of t's ancestors it links to.
func (d *draws) pin(t *table, r int) {
for {
if _, done := d.pinned(t); done {
return
}
if d.npins < len(d.pins) {
d.pins[d.npins] = tablePin{t, r}
d.npins++
} else {
if d.more == nil {
d.more = map[*table]int{}
}
d.more[t] = r
}
if t.parentT == nil {
return
}
t, r = t.parentT, t.parentRow(r)
}
}
// each calls fn for every pinned row, in pin order, the spilled ones by table name.
func (d *draws) each(fn func(t *table, r int)) {
for _, p := range d.pins[:d.npins] {
fn(p.t, p.row)
}
spilled := make([]*table, 0, len(d.more))
for t := range d.more {
spilled = append(spilled, t)
}
sort.Slice(spilled, func(i, j int) bool { return spilled[i].category < spilled[j].category })
for _, t := range spilled {
fn(t, d.more[t])
}
}
// rowOf is the render's row of t: the one pinned, else one drawn inside the
// nearest pinned ancestor — its parent drawn inside that first where the ancestor
// is further up — or over the whole table, and pinned with its ancestors.
func (d *draws) rowOf(t *table) int {
if r, ok := d.pinned(t); ok {
return r
}
r := -1
for a := t.parentT; a != nil && r < 0; a = a.parentT {
if _, ok := d.pinned(a); !ok {
continue
}
if t.parentT != a {
d.rowOf(t.parentT)
}
pr, _ := d.pinned(t.parentT)
r = t.drawUnder(d.s, pr)
}
if r < 0 {
r = t.draw(d.s)
}
d.pin(t, r)
return r
}
// pinRow pins row r of t where it agrees with the rows pinned before it.
func (d *draws) pinRow(t *table, r int) error {
if pr, ok := d.pinned(t); ok && pr != r {
return fmt.Errorf("%s and %s are two rows of %s", t.selectorSpelling(pr), t.selectorSpelling(r), t.path)
}
for a := t.parentT; a != nil; a = a.parentT {
if pa, ok := d.pinned(a); ok && !t.under(r, a, pa) {
return fmt.Errorf("%s is not inside %s", t.selectorSpelling(r), a.selectorSpelling(pa))
}
}
d.pin(t, r)
return nil
}
// selectRow pins the row a selector names.
func (d *draws) selectRow(t *table, sel string) error {
r, err := t.find(sel, d)
if err != nil {
return err
}
return d.pinRow(t, r)
}
// inside keeps the rows of t that sit inside every pinned ancestor.
func (d *draws) inside(t *table, rows []int) []int {
for a := t.parentT; a != nil; a = a.parentT {
pa, ok := d.pinned(a)
if !ok {
continue
}
var kept []int
for _, r := range rows {
if t.under(r, a, pa) {
kept = append(kept, r)
}
}
rows = kept
}
return rows
}
// tableRead is what a reference path reads of a table family: the table its head
// names, the rows its selectors pin, the tables it draws — those it walks with no
// row pinned, and their unpinned ancestors — each selector's spelling, and whether
// it lands on a row rendered whole.
type tableRead struct {
head *table
pins draws
drawn map[*table]bool
sels []tableSel
whole bool
}
// tableSel is one selector on the way: the table it selects a row of, and the path
// as written up to and including it.
type tableSel struct {
t *table
spelling string
}
// tableReadOf replays a reference's selectors through the walk a render uses, so
// two paths pinning one row by different routes compare equal. checkPath proved
// each selector names a row.
func tableReadOf(head node, a arm, leaf node) *tableRead {
t, isTable := head.(*table)
if !isTable {
return nil
}
tr := &tableRead{head: t, drawn: map[*table]bool{}}
_, _ = walkPath(t, a.tail, pathWalk{pins: &tr.pins})
written := a.name[:len(a.name)-len(joinSegments(a.tail))]
cur := t
tr.draws(cur)
for i, seg := range a.tail {
switch d := cur.descendant(seg); {
case isSelector(seg):
tr.sels = append(tr.sels, tableSel{cur, written + joinSegments(a.tail[:i+1])})
case d != nil:
cur = d
tr.draws(cur)
}
}
_, tr.whole = leaf.(*row)
return tr
}
// draws marks t and its ancestors drawn, up to the first the read pins.
func (r *tableRead) draws(t *table) {
for ; t != nil; t = t.parentT {
if _, pinned := r.pins.pinned(t); pinned {
return
}
r.drawn[t] = true
}
}
// joinSegments spells segments as a path: a selector attaches to the name before it.
func joinSegments(segs []string) string {
var b strings.Builder
for _, s := range segs {
if b.Len() > 0 && !isSelector(s) {
b.WriteByte('.')
}
b.WriteString(s)
}
return b.String()
}
// family is the table a chain of parents ends at.
func (t *table) family() *table {
for t.parentT != nil {
t = t.parentT
}
return t
}
// selected is the selector in r on t, or on the nearest ancestor of t it selects.
func (r *tableRead) selected(t *table) (tableSel, bool) {
for ; t != nil; t = t.parentT {
for _, s := range r.sels {
if s.t == t {
return s, true
}
}
}
return tableSel{}, false
}
// checkFamilies refuses reads of one table family in one draw group that cannot
// read one consistent draw: a table rendered whole beside a path into the family,
// a table one read draws that another pins, and two reads pinning different rows.
// The reads come sorted by group and path, so which pair is reported does not vary.
func checkFamilies(reads []pathRead) error {
for i, r := range reads {
if r.tr == nil {
continue
}
for _, o := range reads[:i] {
if o.tr == nil || o.at.group != r.at.group || alternatives(o.at, r.at) || o.tr.head.family() != r.tr.head.family() {
continue
}
if err := checkFamilyPair(o, r); err != nil {
return err
}
}
}
return replayPairs(reads)
}
// replayPairs replays every two reads of one draw group that can render together into
// one draws, the earlier read first. A pin conflicts with one earlier pin, never with a
// combination, so pairs find every conflict a full replay would.
func replayPairs(reads []pathRead) error {
for i, r := range reads {
if r.tr == nil {
continue
}
for _, o := range reads[i+1:] {
if o.tr == nil || o.at.group != r.at.group || alternatives(r.at, o.at) {
continue
}
var d draws
if err := r.tr.replay(&d); err != nil {
return conflict(r, err)
}
if err := o.tr.replay(&d); err != nil {
return conflict(o, err)
}
}
}
return nil
}
func conflict(r pathRead, err error) error {
return fmt.Errorf("%s: %w; select the same rows in every path into the family, or draw them apart with a drawGroup", r.at.route.spelled(r.a.name), err)
}
// replay pins the read's rows into d, where they agree with the rows pinned before.
func (r *tableRead) replay(d *draws) error {
var err error
r.pins.each(func(t *table, row int) {
if err == nil {
err = d.pinRow(t, row)
}
})
return err
}
// checkFamilyPair refuses a table read whole beside a path into its family, and a
// table one read draws that the other pins, since which token renders first would
// then decide the row.
func checkFamilyPair(a, b pathRead) error {
for _, pair := range [][2]pathRead{{a, b}, {b, a}} {
x, y := pair[0], pair[1]
if len(x.a.tail) == 0 && len(y.a.tail) > 0 {
return overlapError(x.at.route, x.a.name, y)
}
if drawn := x.tr.drawnOf(&y.tr.pins); drawn != nil {
if s, ok := y.tr.selected(x.tr.head); ok && len(x.tr.sels) == 0 {
tail := x.a.tail
if s.t != x.tr.head {
tail = append([]string{x.tr.head.category}, tail...)
}
return fmt.Errorf("%s draws %s, which %s selects a row of; write {%s.%s}, or draw them apart with a drawGroup",
x.at.route.spelled(x.a.name), drawn.category, y.at.route.spelled(y.a.name), s.spelling, joinSegments(tail))
}
return fmt.Errorf("%s draws %s, which %s selects a row of; select that row in both, or draw them apart with a drawGroup",
x.at.route.spelled(x.a.name), drawn.category, y.at.route.spelled(y.a.name))
}
}
return nil
}
// drawnOf is a table the read draws that pins holds a row of, if any.
func (r *tableRead) drawnOf(pins *draws) *table {
var found *table
pins.each(func(t *table, _ int) {
if found == nil && r.drawn[t] {
found = t
}
})
return found
}
+83 -203
View File
@@ -1,81 +1,51 @@
// Package fejkdata generates fake data from recursive JSON templates.
//
// Data lives in JSON, not in Go. A generator starts from the shipped data set and
// layers any directories you add; folders and files become a dot-path namespace,
// then generate values by path:
//
// f, _ := fejkdata.New(fejkdata.WithSeed(42))
// f.Fake("sv_SE.address") // "Järvedsvägen 43\n891 77 Järved"
// f.Fake("sv_SE.address.locality") // "Sundbyberg": a second call draws afresh
//
// Several sources merge in order, the last winning a name clash, so custom data
// layers over the built-ins. The JSON template format is documented in the README.
package fejkdata package fejkdata
import ( import (
crand "crypto/rand"
"embed"
"encoding/binary"
"errors" "errors"
"fmt" "fmt"
"io/fs" "io/fs"
"math/rand/v2"
"os"
"reflect" "reflect"
"sort" "sort"
"sync" "sync"
)
//go:embed data "github.com/larvit/fejkdata/internal/datafiles"
var shippedFS embed.FS "github.com/larvit/fejkdata/internal/drawstate"
)
// MaxRepeat caps a repeat, and caps the renders nested repeats multiply to along // MaxRepeat caps a repeat, and caps the renders nested repeats multiply to along
// any one path; the CLI's --repeat shares it. // any one path; the CLI's --repeat shares it.
const MaxRepeat = 1 << 20 const MaxRepeat = 1 << 20
var _ [^uint(0)>>63 - 1]struct{} // 64-bit only, per the README's Decisions // Fails to compile where uint is narrower than 64 bits: docs/decisions.md#64-bit-targets-only
var _ [^uint(0)>>63 - 1]struct{}
// ErrNoData is returned by New when no source is loaded at all. // Generator generates fake data from a namespace tree. Create one with [New].
var ErrNoData = errors.New("no data: WithoutShippedData needs at least one WithDataPath or WithDataFS") // It is safe for concurrent use; a sequence is reproducible only when drawn
// from one goroutine.
// Generator generates fake data from a loaded namespace tree. Create one with [New].
// It is safe for concurrent use; a seeded sequence is reproducible only when drawn
// from one goroutine. The compiled tree is immutable after [New], so List reads it
// without a lock; mu guards the rest, which every other entry point takes.
type Generator struct { type Generator struct {
mu sync.Mutex // mu guards drawState, structs and root, which gains an indexed category on the
rand *session // first call reaching it.
categories map[string]node mu sync.Mutex
root folder // the categories as the node a path walks from, owned here so a walk allocates none drawState *drawstate.State
set drawSet // one Fake's draws, owned here so a walk pinning rows keeps them off the heap root folder // the categories as the node a path walks from, owned here so a walk allocates none
records map[node]recordShape structs map[reflect.Type]structResult
structs map[reflect.Type]structResult
}
// session is one generator's mutable render state: the seeded rng plus the {seq()}
// counters.
type session struct {
*rand.Rand
counters map[string]uint64
}
func (s *session) next(key string) uint64 {
s.counters[key]++
return s.counters[key]
} }
type config struct { type config struct {
seed uint64 seed uint64
seeded bool seeded bool
shipped bool sources []datafiles.Source
sources []dataSource nilModules error
// dataGiven is whether a data option was passed, even WithDataFS(), which loads none.
dataGiven bool
} }
// Option configures a [Generator]. // Option configures a [Generator].
type Option func(*config) type Option func(*config)
// WithSeed makes output reproducible: two generators with the same seed and data // WithSeed seeds the generator, as [New] requires: two generators with the same seed,
// emit identical sequences. // version and data emit identical sequences. For a new seed each run, set
// seed := rand.Uint64() (math/rand/v2), and log it to replay the run.
func WithSeed(seed uint64) Option { func WithSeed(seed uint64) Option {
return func(c *config) { c.seed, c.seeded = seed, true } return func(c *config) { c.seed, c.seeded = seed, true }
} }
@@ -84,175 +54,85 @@ func WithSeed(seed uint64) Option {
// layer several; the last wins a name clash. // layer several; the last wins a name clash.
func WithDataPath(dir string) Option { func WithDataPath(dir string) Option {
return func(c *config) { return func(c *config) {
c.sources = append(c.sources, dataSource{fsys: os.DirFS(dir), label: dir, onDisk: true, path: dir}) c.sources, c.dataGiven = append(c.sources, datafiles.Dir(dir)), true
} }
} }
// WithDataFS layers a data tree held in an [fs.FS], such as an embed.FS of your own. // WithDataFS layers modules, each a data tree held in an [fs.FS], over what is loaded
func WithDataFS(fsys fs.FS) Option { // before them, in order. A module may be a data package's FS, such as one of
return func(c *config) { c.sources = append(c.sources, dataSource{fsys: fsys}) } // github.com/larvit/fejkdata/data's Modules, or an embed.FS of your own. WithDataFS()
// loads no data.
func WithDataFS(modules ...fs.FS) Option {
return func(c *config) {
for i, fsys := range modules {
if fsys == nil {
c.nilModules = errors.Join(c.nilModules, fmt.Errorf("fejkdata: WithDataFS module %d is nil", i+1))
continue
}
c.sources = append(c.sources, datafiles.FS(fsys))
}
c.dataGiven = true
}
} }
// WithoutShippedData leaves the shipped data set out, so only the sources given var errNoSeed = errors.New(`fejkdata: no seed: add fejkdata.WithSeed(n), where the same n repeats a run's output; for a new seed each run, set n := rand.Uint64() (import "math/rand/v2") and log n to replay the run`)
// with [WithDataPath] and [WithDataFS] load.
func WithoutShippedData() Option {
return func(c *config) { c.shipped = false }
}
// New builds a generator from the shipped data set and the options' sources, merged var errNoData = errors.New(`fejkdata: no data: add fejkdata.WithDataFS(data.Modules()...) (import "github.com/larvit/fejkdata/data") for all shipped data, WithDataFS/WithDataPath for your own, or WithDataFS() for none`)
// in order with the last winning a name clash. Each JSON file becomes a category
// named after the file (address.json -> "address") and each subdirectory a // ErrLoad marks data that fails to load: in New, or on the first call reaching a category
// namespace segment. It errors on a missing directory, invalid JSON, invalid data, // of an indexed source.
// or no data at all. var ErrLoad = errors.New("data fails to load")
// loadError is a load's error, unchanged in its text, matching ErrLoad.
type loadError struct{ error }
func (e loadError) Unwrap() error { return e.error }
func (loadError) Is(target error) bool { return target == ErrLoad }
// New builds a generator from the options' sources, merged in order with the last
// winning a name clash; it fails where no [WithSeed] gives the seed, or no [WithDataFS]
// or [WithDataPath] names the data to load. Each JSON file becomes a category named
// after the file (address.json -> "address") and each subdirectory a namespace segment.
// It errors on a missing directory, invalid JSON or invalid data. Given only
// WithDataFS() with no modules, New loads no data, and the generator renders templates
// that read none. A source whose .fejkdata.json carries an index, as each shipped
// module's does, loads each category on the first call reaching it. Data failing to
// load, in New or on that first call, gives an error matching [ErrLoad].
func New(opts ...Option) (*Generator, error) { func New(opts ...Option) (*Generator, error) {
c := config{shipped: true} var c config
for _, opt := range opts { for _, opt := range opts {
opt(&c) opt(&c)
} }
var sources []dataSource if err := c.misuse(); err != nil {
if c.shipped { return nil, err
sources = append(sources, dataSource{fsys: shippedFS, root: "data"})
} }
sources = append(sources, c.sources...) root, err := loadSources(c.sources)
if len(sources) == 0 {
return nil, fmt.Errorf("fejkdata: %w", ErrNoData)
}
cats, err := loadData(sources)
if err != nil { if err != nil {
return nil, fmt.Errorf("fejkdata: %w", err) return nil, fmt.Errorf("fejkdata: %w", loadError{err})
} }
rng, err := newRand(c.seed, c.seeded) return &Generator{drawState: drawstate.New(c.seed), root: root}, nil
if err != nil {
return nil, fmt.Errorf("fejkdata: %w", err)
}
return &Generator{rand: rng, categories: cats}, nil
} }
// List returns the sorted dotted paths Fake can render: every category, the dotted // List returns the sorted dotted paths Fake renders: each category and every field,
// fields within a template, and folder segments. A choice consumes no segment, so a // column and linked table below it, a direct descent at a time, listing a level
// path continues through one only where every variant carries it, which is the // carrying a repeat but nothing below it. A choice consumes no segment, so a path
// rule Fake applies too: List is the set of paths Fake accepts. // continues through one only where every variant carries it.
func (f *Generator) List() []string { func (f *Generator) List() []string {
var out []string f.mu.Lock()
for _, name := range sortedNames(f.categories) { defer f.mu.Unlock()
for _, p := range paths(f.categories[name]) { out := paths(&f.root, false)
out = append(out, join(name, p))
}
}
sort.Strings(out) sort.Strings(out)
return out return out
} }
// paths lists the dot paths addressable from n, relative to it, where "" is n // misuse names every mistake in the options, before any data loads.
// itself. A folder has no value of its own, so it contributes only its children's. func (c config) misuse() error {
func paths(n node) []string { var err error
switch n := n.(type) { if !c.dataGiven {
case *folder: err = errNoData
var out []string
for _, name := range sortedNames(n.children) {
for _, p := range paths(n.children[name]) {
out = append(out, join(name, p))
}
}
return out
case *template:
out := []string{""}
for _, name := range sortedNames(n.fields) {
if isRef(name) { // a binding, not a path segment
continue
}
for _, p := range paths(n.fields[name]) {
out = append(out, join(name, p))
}
}
return out
case *null:
return []string{""}
case *table:
return tablePaths(n)
case *column, *row:
return []string{""}
case *choice:
out := []string{""}
for p := range n.shared {
out = append(out, p)
}
return out
} }
return nil if !c.seeded {
} err = errors.Join(err, errNoSeed)
}
// tablePaths is a table's columns, then each table linked to it under its name: the return errors.Join(err, c.nilModules)
// direct descents, a step at a time.
func tablePaths(t *table) []string {
out := append([]string{""}, t.columns...)
sort.Strings(out[1:])
children := make([]string, 0, len(t.children))
for name := range t.children {
children = append(children, name)
}
sort.Strings(children)
for _, name := range children {
for _, p := range paths(t.children[name]) {
out = append(out, join(name, p))
}
}
return out
}
// sharedPaths is the sub-paths every item carries — the only ones a path may step
// through a choice to reach. It intersects, bailing as soon as the set
// is empty, which is immediate for a choice of plain strings.
func sharedPaths(items []node) map[string]bool {
shared := subPaths(items[0])
for _, it := range items[1:] {
if len(shared) == 0 {
return nil
}
next := subPaths(it)
for p := range shared {
if !next[p] {
delete(shared, p)
}
}
}
return shared
}
// subPaths is paths(n) as a set, without the empty path that means n itself.
func subPaths(n node) map[string]bool {
out := map[string]bool{}
for _, p := range paths(n) {
if p != "" {
out[p] = true
}
}
return out
}
func join(prefix, name string) string {
switch {
case prefix == "":
return name
case name == "":
return prefix
}
return prefix + "." + name
}
// randomBytes seeds an unseeded generator.
var randomBytes = crand.Read
func newRand(seed uint64, seeded bool) (*session, error) {
var r *rand.Rand
if seeded {
r = rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15))
} else {
var b [16]byte
if _, err := randomBytes(b[:]); err != nil {
return nil, fmt.Errorf("seeding from crypto/rand: %w", err)
}
r = rand.New(rand.NewPCG(binary.LittleEndian.Uint64(b[:8]), binary.LittleEndian.Uint64(b[8:])))
}
return &session{Rand: r, counters: map[string]uint64{}}, nil
} }
+73 -14
View File
@@ -3,11 +3,18 @@ package fejkdata
import ( import (
"errors" "errors"
"io/fs" "io/fs"
"os"
"path/filepath"
"strings" "strings"
"testing" "testing"
"testing/fstest"
"github.com/larvit/fejkdata/data"
"github.com/larvit/fejkdata/internal/drawstate"
"github.com/larvit/fejkdata/internal/jsonvalue"
) )
func withShipped() Option { return WithDataFS(data.Modules()...) }
// newGenerator creates a generator over a single data directory, failing on // newGenerator creates a generator over a single data directory, failing on
// error. Most tests load one dir; newGeneratorN loads several (last-loaded wins // error. Most tests load one dir; newGeneratorN loads several (last-loaded wins
// on conflicts). // on conflicts).
@@ -18,17 +25,28 @@ func newGenerator(t *testing.T, dir string, opts ...Option) *Generator {
func newGeneratorN(t *testing.T, dirs []string, opts ...Option) *Generator { func newGeneratorN(t *testing.T, dirs []string, opts ...Option) *Generator {
t.Helper() t.Helper()
all := []Option{WithoutShippedData()} var all []Option
for _, dir := range dirs { for _, dir := range dirs {
all = append(all, WithDataPath(dir)) all = append(all, WithDataPath(dir))
} }
f, err := New(append(all, opts...)...) f, err := New(append(append(all, WithSeed(1)), opts...)...)
if err != nil { if err != nil {
t.Fatalf("New(%q): %v", dirs, err) t.Fatalf("New(%q): %v", dirs, err)
} }
return f return f
} }
// newBareModule loads the shipped module in data/<module> without its manifest, so its
// categories sit at the root, failing on error.
func newBareModule(t *testing.T, module string, opts ...Option) *Generator {
t.Helper()
f, err := New(append([]Option{WithDataFS(withoutManifest{os.DirFS(filepath.Join("data", module))}), WithSeed(1)}, opts...)...)
if err != nil {
t.Fatalf("New(%s without its manifest): %v", module, err)
}
return f
}
// fake generates a value, failing the test on error. // fake generates a value, failing the test on error.
func fake(t *testing.T, f *Generator, path string) string { func fake(t *testing.T, f *Generator, path string) string {
t.Helper() t.Helper()
@@ -40,31 +58,32 @@ func fake(t *testing.T, f *Generator, path string) string {
} }
func TestNewMissingDirectory(t *testing.T) { func TestNewMissingDirectory(t *testing.T) {
_, err := New(WithoutShippedData(), WithDataPath("data/de_DE")) _, err := New(WithSeed(1), WithDataPath("data/de_DE"))
if err == nil || !strings.Contains(err.Error(), "de_DE") || !errors.Is(err, fs.ErrNotExist) { if err == nil || !strings.Contains(err.Error(), "de_DE") || !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("New(missing) error = %v, want the real error, naming the path", err) t.Fatalf("New(missing) error = %v, want the real error, naming the path", err)
} }
} }
func TestNewRejectsAnEmptyDataPath(t *testing.T) { func TestNewRejectsAnEmptyDataPath(t *testing.T) {
_, err := New(WithoutShippedData(), WithDataPath("")) _, err := New(WithSeed(1), WithDataPath(""))
if err == nil || !strings.Contains(err.Error(), "empty") { if err == nil || !strings.Contains(err.Error(), "empty") {
t.Fatalf("New(WithDataPath(\"\")) = %v, want the empty path named", err) t.Fatalf("New(WithDataPath(\"\")) = %v, want the empty path named", err)
} }
} }
func TestNewReportsAnEntropyFailure(t *testing.T) { func TestNewWithoutASeedFailsNamingWithSeed(t *testing.T) {
saved := randomBytes for name, opts := range map[string][]Option{"no option": nil, "only data": {WithDataFS()}} {
randomBytes = func([]byte) (int, error) { return 0, errors.New("no entropy") } if _, err := New(opts...); err == nil || !strings.Contains(err.Error(), "WithSeed(") {
defer func() { randomBytes = saved }() t.Errorf("New(%s) = %v, want it refused, naming WithSeed", name, err)
_, err := New(WithoutShippedData(), WithDataFS(fstest.MapFS{"w.json": {Data: []byte(`"x"`)}})) }
if err == nil || !strings.Contains(err.Error(), "no entropy") { }
t.Fatalf("New() without entropy = %v, want the failure reported", err) if _, err := New(WithDataFS(nil)); err == nil || !strings.Contains(err.Error(), "WithSeed(") || !strings.Contains(err.Error(), "module 1 is nil") {
t.Errorf("New(WithDataFS(nil)) = %v, want both the missing seed and the nil module named", err)
} }
} }
func TestWithSeedIsDeterministic(t *testing.T) { func TestWithSeedIsDeterministic(t *testing.T) {
a, b := newGenerator(t, "data", WithSeed(42)), newGenerator(t, "data", WithSeed(42)) a, b := shipped(t, WithSeed(42)), shipped(t, WithSeed(42))
for i := 0; i < 50; i++ { for i := 0; i < 50; i++ {
if x, y := fake(t, a, "sv_SE.person"), fake(t, b, "sv_SE.person"); x != y { if x, y := fake(t, a, "sv_SE.person"), fake(t, b, "sv_SE.person"); x != y {
t.Fatalf("same seed diverged at %d: %q != %q", i, x, y) t.Fatalf("same seed diverged at %d: %q != %q", i, x, y)
@@ -73,7 +92,7 @@ func TestWithSeedIsDeterministic(t *testing.T) {
} }
func TestDifferentSeedsDiffer(t *testing.T) { func TestDifferentSeedsDiffer(t *testing.T) {
a, b := newGenerator(t, "data", WithSeed(1)), newGenerator(t, "data", WithSeed(2)) a, b := shipped(t, WithSeed(1)), shipped(t, WithSeed(2))
for i := 0; i < 50; i++ { for i := 0; i < 50; i++ {
if fake(t, a, "en_US.person") != fake(t, b, "en_US.person") { if fake(t, a, "en_US.person") != fake(t, b, "en_US.person") {
return return
@@ -92,3 +111,43 @@ func fakeTemplate(t *testing.T, f *Generator, s string) string {
} }
return got return got
} }
// engine builds a seeded generator with no loaded categories, for rendering tests.
func engine(seed uint64) *Generator {
return &Generator{drawState: drawstate.New(seed)}
}
// parse unmarshals a JSON template fragment into its dynamic form.
func parse(t *testing.T, s string) jsonvalue.Value {
t.Helper()
v, err := jsonvalue.Decode([]byte(s))
if err != nil {
t.Fatalf("parse %q: %v", s, err)
}
return v
}
// resolved compiles a JSON fragment and resolves it against no data.
func resolved(t *testing.T, s string) (node, error) {
t.Helper()
n, err := compile(parse(t, s))
if err != nil {
return nil, err
}
return n, resolveInlineTemplates(inlineNodes(n, "template"), nil)
}
// compiled parses, compiles and resolves a JSON fragment into a node.
func compiled(t *testing.T, s string) node {
t.Helper()
n, err := resolved(t, s)
if err != nil {
t.Fatalf("compile %q: %v", s, err)
}
return n
}
func mustRender(t *testing.T, f *Generator, s string) string {
t.Helper()
return renderOnce(f.drawState, compiled(t, s))
}
+205
View File
@@ -0,0 +1,205 @@
package fejkdata
import (
"fmt"
"strings"
"github.com/larvit/fejkdata/internal/builtinfunc"
"github.com/larvit/fejkdata/internal/grammar"
)
// checkFunc validates a call at compile time: a known builtin, its args, and, for calc, the
// operands held in fields.
func checkFunc(tok grammar.Token, fields map[string]node) error {
err := builtinfunc.Check(tok.Fn, tok.Args)
if err == nil && tok.Fn == builtinfunc.CalcName {
err = checkCalcFields(tok.Args, fields)
}
if err != nil {
return fmt.Errorf("token {%s}: %w", tok.Body, err)
}
return nil
}
func parseChecked(format string, fields map[string]node) ([]grammar.Token, []unboundRead, error) {
toks, err := grammar.ParseFormat(format)
if err != nil {
return nil, nil, err
}
unbound, err := checkTokens(toks, fields)
return toks, unbound, err
}
// unboundRead is a token reading a head no field holds, which only a name bound around the
// template can answer; err is the refusal where none does, and body is the token's.
type unboundRead struct {
head, body string
err error
}
// checkTokens proves every token's grammar, functions and field paths, and returns the reads
// whose head no field holds, for bindNames to look up among the names.
func checkTokens(toks []grammar.Token, fields map[string]node) ([]unboundRead, error) {
var unbound []unboundRead
for _, t := range toks {
u, err := checkToken(t, fields)
if err != nil {
return nil, err
}
unbound = append(unbound, u...)
}
return unbound, nil
}
func checkToken(t grammar.Token, fields map[string]node) ([]unboundRead, error) {
switch t.Kind {
case grammar.BuiltinCall:
if err := checkFunc(t, fields); err != nil {
return nil, err
}
return checkReads(t, fields, true)
case grammar.NameBind:
return nil, checkBind(t, fields)
case grammar.PathRead:
return checkReads(t, fields, false)
}
return nil, nil
}
// checkReads proves each name t reads, an arm or an operand, is a reference or a path into a
// field, returning those whose head no field holds.
func checkReads(t grammar.Token, fields map[string]node, operands bool) ([]unboundRead, error) {
var unbound []unboundRead
names := tokenReads(t)
for _, name := range names {
if grammar.IsRef(name) {
if _, _, err := grammar.RefShape(name); err != nil {
return nil, fmt.Errorf("token {%s}: %w", t.Body, err)
}
continue // resolveRefs checks its target once the tree is assembled
}
missing, err := checkArm(name, fields, !operands && len(names) == 1)
if err == nil {
continue
}
err = fmt.Errorf("token {%s}: %w", t.Body, err)
if !missing {
return nil, err
}
unbound = append(unbound, unboundRead{head: splitArm(name, nil).head, body: t.Body, err: err})
}
return unbound, nil
}
// checkBind proves a {x as name} token is spelled once, binds a reference or a path into a
// field, and names a valid name that is no option.
func checkBind(t grammar.Token, fields map[string]node) error {
ref, name := t.BoundRef, t.Bound
if trimmed := strings.TrimSpace(ref) + grammar.AsWord + strings.TrimSpace(name); trimmed != t.Body {
return fmt.Errorf("token {%s}: write {%s}", t.Body, trimmed)
}
if err := checkBound(ref, fields); err != nil {
return fmt.Errorf("token {%s}: %w", t.Body, err)
}
if name == "" {
return fmt.Errorf("token {%s}: a binding names nothing; write {%s as n}", t.Body, ref)
}
if isOption(name) {
return fmt.Errorf("token {%s}: %q is an option and can never be a name; rename it", t.Body, name)
}
if err := grammar.CheckIdentifier(name); err != nil {
return fmt.Errorf("token {%s}: name %w", t.Body, err)
}
return nil
}
// checkBound proves what a binding binds is a reference or a path into a field of its binder.
func checkBound(ref string, fields map[string]node) error {
if grammar.IsRef(ref) {
_, _, err := grammar.RefShape(ref)
return err
}
missing, err := checkArm(ref, fields, false)
if missing {
return fmt.Errorf("%w; a binding names a field, or a reference, which starts with /, . or ..", err)
}
return err
}
// checkArm validates one sibling name or path against a template's fields.
// wholeToken says the name is the token's entire body, so {/name} would render
// the same value and can be offered as the reference spelling. missing says no
// field holds the head, which a name may yet answer.
func checkArm(name string, fields map[string]node, wholeToken bool) (missing bool, err error) {
a := splitArm(name, nil)
if err := checkSegments(a); err != nil {
return false, err
}
field, ok := fields[a.head]
if !ok {
if a.head == "" {
return false, fmt.Errorf("a name is never empty, so this token can name no field")
}
if isOption(a.head) {
return false, fmt.Errorf("%q is an option and can never be a field", a.head)
}
if len(fields) == 0 {
hint := ""
if wholeToken && hintableRef(name) {
hint = fmt.Sprintf(" — write {/%s} to reference the data", name)
}
return true, fmt.Errorf("no field %q; a token names a sibling field, and this template has none%s", a.head, hint)
}
return true, fmt.Errorf("no field %q", a.head)
}
if err := provePath(field, a.tail, a.head, 0); err != nil {
return false, fmt.Errorf("field %q: %w", a.head, err)
}
return false, nil
}
// hintableRef reports whether {/name} is a reference the grammar accepts, so the
// hint never names a spelling that fails too.
func hintableRef(name string) bool {
return grammar.CheckPathIdentifiers(name) == nil
}
// tokenReads lists the names t reads: a read's arms, or the operands a call's builtin reads.
func tokenReads(t grammar.Token) []string {
if t.Kind == grammar.BuiltinCall {
return builtinfunc.Operands(t.Fn, t.Args)
}
return t.Arms
}
// formatOps is a compiled format: its ops, and the size of its literal text.
type formatOps struct {
ops []op
grow int
}
// compileFormat compiles t's parsed format against its resolved references and its names'
// targets; a {ref as n} token compiles to no op. checkTokens proved every token valid.
func compileFormat(t *template, targets map[*nameBinding]nameTarget) (formatOps, error) {
var c formatOps
for _, tok := range t.tokens {
o := op{Token: tok}
var err error
switch tok.Kind {
case grammar.LiteralRun:
c.grow += len(tok.Lit)
case grammar.BuiltinCall:
o.call = builtinfunc.Prep(tok.Fn, tok.Args)
o.operands, err = t.compileArms(builtinfunc.Operands(tok.Fn, tok.Args), targets)
case grammar.PathRead:
o.arms, err = t.compileArms(tok.Arms, targets)
case grammar.NameBind:
continue
}
if err != nil {
return c, fmt.Errorf("token {%s}: %w", tok.Body, err)
}
c.ops = append(c.ops, o)
}
return c, nil
}
+54
View File
@@ -0,0 +1,54 @@
package fejkdata
import "testing"
// TestSeededOutputIsStable pins the exact bytes a seeded generator emits for each
// piece of the token grammar, so a change to how a format is scanned or rendered
// cannot quietly shift the rng stream that reproducibility depends on.
func TestSeededOutputIsStable(t *testing.T) {
dir := writeFiles(t, map[string]string{
"alt.json": `{"format":"{a|b}","a":"A","b":"B"}`,
"calc.json": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
"classes.json": `"{digits(2)}-{int(1,9)}{int(1,9)}-{upper(2)}-{lower(2)}"`,
"escapes.json": `{"format":"01Aa#{x}","x":"!"}`,
"funcs.json": `"{hex(6)} {int(10,99)} {float(0,1,3)} {nanoid(5)} {seq()}"`,
"municipality.json": `{"format":"{name}","rows":"municipality.tsv","key":"code","name":"name","parent":"region"}`,
"municipality.tsv": "code\tname\tregion\n0180\tStockholm\t01\n0184\tSolna\t01\n1280\tMalmö\t12\n1281\tLund\t12\n",
"named.json": `{"format":"{p as q}{q.a}{q.b}","p":[{"format":"x","a":"1","b":"2"},{"format":"y","a":"A","b":"B"},{"format":"z","a":"a","b":"b"}]}`,
"nested.json": `{"format":"{outer}","outer":{"format":"{inner}-{digits(2)}","inner":"i"}}`,
"paths.json": `{"format":"{p.a}{p.b}","p":[{"format":"x","a":"1","b":"2"},{"format":"y","a":"A","b":"B"},{"format":"z","a":"a","b":"b"}]}`,
"ref.json": `"see {/alt}"`,
"refpath.json": `"{/region.municipality.name} in {/region.name}"`,
"region.json": `{"format":"{name}","rows":"region.tsv","key":"code","name":"name","weight":"people"}`,
"region.tsv": "code\tname\tpeople\n01\tStockholms län\t3\n12\tSkåne län\t1\n",
"repeat.json": `{"format":"{w}","repeat":4,"separator":",","w":["x","y","z"]}`,
"sums.json": `{"format":"9{d}{luhn()} {e}{ean()} {m}{mod11()}","d":"012345678901234","e":"123456789012","m":"12345678"}`,
"weights.json": `[{"format":"big","weight":9},"tiny"]`,
})
want := map[string][]string{
"alt": {"A", "A", "B", "A"},
"calc": {"100.00 x 3 = 700.00", "5.00 x 7 = 35.00", "5.00 x 3 = 35.00", "100.00 x 2 = 139.93"},
"classes": {"49-74-VY-gj", "38-68-RP-xo", "47-68-IB-hs", "91-89-LP-gk"},
"escapes": {"01Aa#!", "01Aa#!", "01Aa#!", "01Aa#!"},
"funcs": {"0016a2 33 0.649 k4Kwj 1", "a00c07 84 0.175 6UjGv 2", "f70eb8 12 0.390 p5p6g 3", "9048a3 90 0.531 I3Aqv 4"},
"named": {"ab", "ab", "AB", "AB"},
"nested": {"i-73", "i-23", "i-67", "i-85"},
"paths": {"aB", "aB", "ab", "AB"},
"paths.p.a": {"A", "a", "a", "A"},
"ref": {"see A", "see A", "see A", "see B"},
"refpath": {"Solna in Skåne län", "Solna in Stockholms län", "Solna in Stockholms län", "Stockholm in Stockholms län"},
"region.municipality[0184].name": {"Solna", "Solna", "Solna", "Solna"},
"region[12].municipality.name": {"Malmö", "Malmö", "Lund", "Malmö"},
"repeat": {"z,y,z,y", "z,z,y,y", "z,z,x,z", "x,z,y,y"},
"sums": {"90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780"},
"weights": {"big", "big", "big", "big"},
}
for path := range want {
f := newGenerator(t, dir, WithSeed(42))
for i, expect := range want[path] {
if got := fake(t, f, path); got != expect {
t.Errorf("%s draw %d = %q, want %q", path, i, got, expect)
}
}
}
}
+132
View File
@@ -0,0 +1,132 @@
package fejkdata
import (
"regexp"
"strings"
"testing"
)
func TestStringIsAFormat(t *testing.T) {
f := engine(1)
for src, want := range map[string]string{
`"Malmö"`: "Malmö",
`"100 Main St, Apt 1A #0"`: "100 Main St, Apt 1A #0",
`"{{x}}"`: "{x}",
} {
if got := mustRender(t, f, src); got != want {
t.Errorf("render(%s) = %q, want %q", src, got, want)
}
}
if got := mustRender(t, f, `"{digits(3)}"`); !regexp.MustCompile(`^[0-9]{3}$`).MatchString(got) {
t.Errorf(`render("{digits(3)}") = %q, want three digits`, got)
}
if _, err := resolved(t, `"{x}"`); err == nil || !strings.Contains(err.Error(), `no field "x"`) {
t.Errorf(`compile("{x}") = %v, want a no-field error`, err)
}
}
func TestTextIsLiteral(t *testing.T) {
if got := mustRender(t, engine(1), `{"format":"100 Main St #1 {x}","x":"A"}`); got != "100 Main St #1 A" {
t.Fatalf("text = %q, want it verbatim", got)
}
}
func TestBraceEscapes(t *testing.T) {
f := engine(1)
for src, want := range map[string]string{
`{"format":"{{","x":"v"}`: "{",
`{"format":"}}","x":"v"}`: "}",
`{"format":"{{x}}","x":"v"}`: "{x}",
`{"format":"{{{x}}}","x":"v"}`: "{v}",
`{"format":"a{{{{b}}}}","x":"v"}`: "a{{b}}",
} {
if got := mustRender(t, f, src); got != want {
t.Errorf("render(%s) = %q, want %q", src, got, want)
}
}
for src, want := range map[string]string{
`"x}y"`: "}}",
`"}"`: "}}",
`{"format":"{a{b}","a":"Q"}`: "'{'",
`"{x"`: "unterminated",
} {
if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("compile(%s) = %v, want an error mentioning %s", src, err, want)
}
}
}
func TestTokenSubstitution(t *testing.T) {
if got := mustRender(t, engine(1), `{"format":"{x}sson","x":"Erik"}`); got != "Eriksson" {
t.Fatalf("token = %q, want Eriksson", got)
}
}
func TestAlternationDrawsOneField(t *testing.T) {
f := engine(3)
seen := map[string]bool{}
for i := 0; i < 100; i++ {
seen[mustRender(t, f, `{"format":"{a|b}","a":"A","b":"B"}`)] = true
}
if !seen["A"] || !seen["B"] || len(seen) != 2 {
t.Fatalf("alternation produced %v, want both A and B", seen)
}
}
func TestAlternationThreeWay(t *testing.T) {
f := engine(4)
seen := map[string]bool{}
for i := 0; i < 200; i++ {
seen[mustRender(t, f, `{"format":"{a|b|c}","a":"A","b":"B","c":"C"}`)] = true
}
if !seen["A"] || !seen["B"] || !seen["C"] || len(seen) != 3 {
t.Fatalf("3-way alternation produced %v, want A, B and C", seen)
}
}
func TestHashIsLiteral(t *testing.T) {
f := engine(1)
cases := map[string]string{
`{"format":"","x":"v"}`: "",
`{"format":"#","x":"v"}`: "#",
`{"format":"##","x":"v"}`: "##",
`{"format":"#0#1#A#a","x":"v"}`: "#0#1#A#a",
`{"format":"#{x}","x":"v"}`: "#v",
}
for tmpl, want := range cases {
if got := mustRender(t, f, tmpl); got != want {
t.Errorf("render(%s) = %q, want %q", tmpl, got, want)
}
}
}
func TestMultibyteFormat(t *testing.T) {
// Scanning is rune-aware: multibyte literals coexist with class chars and
// tokens without corrupting indices.
got := mustRender(t, engine(2), `{"format":"Öster{x}-{digits(1)}å","x":"väg"}`)
if !regexp.MustCompile(`^Österväg-[0-9]å$`).MatchString(got) {
t.Fatalf("multibyte format = %q", got)
}
}
func TestFormatCompileErrors(t *testing.T) {
for _, bad := range []string{
`"{x"`, // unterminated brace
`"x}y"`, // a lone } must be written }}
`{"format":"{a{b}","a":"Q"}`, // a brace inside a token
`"{x}"`, // a bare string has no fields to name
`{"format":"{y}","x":"Q"}`, // token names a missing field
`"{}"`, // empty token name
`{"format":"{a|}","a":"Q"}`, // empty alternation segment
`"{luhn(}"`, // malformed function token
`"{lower()}"`, // wrong arity
`"{nope()}"`, // unknown function
`"{luhn(x)}"`, // function given args it takes none of
`"{int(1)}"`, // wrong arity
`"{float(1,2)}"`, // wrong arity
} {
if _, err := resolved(t, bad); err == nil {
t.Errorf("compile(%s) = nil error, want error", bad)
}
}
}
+1 -1
View File
@@ -1,3 +1,3 @@
module gitea.larvit.se/larvit/fejkdata module github.com/larvit/fejkdata
go 1.22 go 1.22
+84 -182
View File
@@ -2,128 +2,99 @@ package fejkdata
import ( import (
"fmt" "fmt"
"sort" "slices"
"github.com/larvit/fejkdata/internal/invariant"
) )
// walkNodes calls fn once per contained node, passing the dot path that reaches it, // eachNode visits n and every node inside it once, and passes fn each node with the label an
// visiting keys in sorted order so which of several broken nodes gets reported does // error uses for it: the dot path reaching it, then ", line N" for a table cell. It never
// not depend on map iteration. // crosses a reference edge — a {/path} reference is skipped — so a single inline node is
func walkNodes(root map[string]node, fn func(path string, n node) error) error { // walked on its own.
for _, name := range sortedNames(root) { func eachNode(n node, label string, fn func(label string, n node) error) error {
if err := eachNode(root[name], name, fn); err != nil {
return err
}
}
return nil
}
// eachNode visits n and every node contained within it once, passing the dot path
// that reaches each. It never crosses a reference edge — a bound {/path} field is
// skipped — so a single inline node is walked on its own.
func eachNode(n node, path string, fn func(path string, n node) error) error {
seen := map[node]bool{} seen := map[node]bool{}
var visit func(string, node) error var visit func(string, node) error
visit = func(path string, m node) error { visit = func(label string, m node) error {
if m == nil || seen[m] { if m == nil || seen[m] {
return nil return nil
} }
seen[m] = true seen[m] = true
if err := fn(path, m); err != nil { if err := fn(label, m); err != nil {
return err return err
} }
for _, c := range contained(m) { for _, c := range contained(m) {
if err := visit(join(path, c.name), c.node); err != nil { if err := visit(c.labelIn(label), c.node); err != nil {
return err return err
} }
} }
return nil return nil
} }
return visit(path, n) return visit(label, n)
} }
// namedNode is a contained child and the segment reaching it; a choice's items carry // containedNode is a contained child and the segment reaching it; a choice's items carry
// no segment, matching how a dot path steps over a choice. // no segment, matching how a dot path steps over a choice.
type namedNode struct { type containedNode struct {
name string name string
node node node node
line int // a table cell's line in its rows file, else 0
} }
func contained(n node) []namedNode { func (c containedNode) labelIn(label string) string {
if c.line > 0 {
return fmt.Sprintf("%s, line %d", label, c.line)
}
return join(label, c.name)
}
func contained(n node) []containedNode {
switch n := n.(type) { switch n := n.(type) {
case *folder: case *folder:
return named(n.children) return containedByName(n.children)
case *choice: case *choice:
out := make([]namedNode, len(n.items)) out := make([]containedNode, len(n.items))
for i, it := range n.items { for i, it := range n.items {
out[i] = namedNode{node: it} out[i] = containedNode{node: it}
} }
return out return out
case *template: case *template:
return named(n.fields) return containedByName(n.fields)
case *table: case *table:
return append([]namedNode{{node: n.format}}, named(n.fields)...) return append([]containedNode{{node: n.formatTemplate}}, containedByName(n.formatTemplate.fields)...)
case *column: case *tableColumn:
if len(n.t.tokens) == 0 { if len(n.t.cellTemplates) == 0 {
return nil return nil
} }
var out []namedNode var out []containedNode
for r := 0; r < n.t.rows(); r++ { for r := 0; r < n.t.rows.Len(); r++ {
if cell := n.t.cellNode(r, n.i); cell != nil { if cell := n.t.cellTemplate(r, n.i); cell != nil {
out = append(out, namedNode{node: cell}) out = append(out, containedNode{node: cell, line: r + 2})
} }
} }
return out return out
default: case *nullItem, *tableRow:
return nil return nil
default:
panic(invariant.Broken("contained has no case for node %T", n))
} }
} }
// named skips a bound {/path} key: it is a render edge, not containment, so using func containedByName(m map[string]node) []containedNode {
// it as a path segment would report a node under a path that does not reach it. Only out := make([]containedNode, 0, len(m))
// a template's fields hold bindings — loadDir skips a dot-prefixed entry, so a
// folder's children never carry the prefix — so this one skip serves both.
func named(m map[string]node) []namedNode {
out := make([]namedNode, 0, len(m))
for _, name := range sortedNames(m) { for _, name := range sortedNames(m) {
if isRef(name) { out = append(out, containedNode{name: name, node: m[name]})
continue
}
out = append(out, namedNode{name: name, node: m[name]})
} }
return out return out
} }
func sortedNames(m map[string]node) []string {
names := make([]string, 0, len(m))
for name := range m {
names = append(names, name)
}
sort.Strings(names)
return names
}
// renderEdge is a child a node renders into, labelled by what reaches it (a field // renderEdge is a child a node renders into, labelled by what reaches it (a field
// name, reference, or choice index) for a readable cycle report. operand names // name, reference, or choice index) for a readable cycle report.
// the builtin when the label is its operand rather than a token, so an error can
// name it the way the author wrote it.
type renderEdge struct { type renderEdge struct {
to node to node
label string label string
operand string
} }
// reached names an edge as the author spelled it, the vocabulary boundReaders uses // renderEdges lists the children rendering n recurses into, mirroring render.
// for the sibling fence.
func (e renderEdge) reached() string {
if e.operand != "" {
return fmt.Sprintf("%s operand %q", e.operand, e.label)
}
return "{" + e.label + "}"
}
// renderEdges lists the children rendering n recurses into, mirroring expand: a
// choice's items, and a template's field/reference tokens plus its operands. A
// folder renders nothing, so it has no edges.
func renderEdges(n node) []renderEdge { func renderEdges(n node) []renderEdge {
switch n := n.(type) { switch n := n.(type) {
case *choice: case *choice:
@@ -134,120 +105,40 @@ func renderEdges(n node) []renderEdge {
return es return es
case *template: case *template:
var es []renderEdge var es []renderEdge
add := func(name, operand string) { for _, o := range n.compiled.ops {
a := splitArm(name, n.refs) for _, a := range slices.Concat(o.operands, o.arms) {
c, ok := n.fields[a.key] for _, leaf := range a.leaves {
if !ok { es = append(es, renderEdge{leaf, a.spelling})
return }
}
for _, leaf := range pathLeaves(c, a.tail) {
es = append(es, renderEdge{leaf, name, operand})
} }
} }
_ = eachToken(n.format, func(t ftoken) error {
if t.kind != 'b' {
return nil
}
if fn, _, isFunc := funcCall(t.body); isFunc {
for _, operand := range tokenOperands(t.body) {
add(operand, fn)
}
return nil
}
for _, name := range splitOutside(t.body, '|') {
add(name, "")
}
return nil
})
return es return es
case *table: case *table:
return []renderEdge{{to: n.format, label: "format"}} return []renderEdge{{to: n.formatTemplate, label: "format"}}
case *row: case *tableRow:
return []renderEdge{{to: n.t.format, label: "format"}} return []renderEdge{{to: n.t.formatTemplate, label: "format"}}
case *column: case *tableColumn:
var es []renderEdge var es []renderEdge
for _, c := range contained(n) { for _, c := range contained(n) {
es = append(es, renderEdge{to: c.node, label: n.t.columns[n.i]}) es = append(es, renderEdge{to: c.node, label: n.name()})
} }
return es return es
default: case *folder, *nullItem:
return nil return nil
default:
panic(invariant.Broken("renderEdges has no case for node %T", n))
} }
} }
// pathLeaves lists what a token's dotted tail renders: a choice on the way type renderCounts map[node]int
// contributes every variant, since any of them may be the one drawn. checkPath has
// already proved the tail resolves in every variant.
func pathLeaves(n node, tail []string) []node {
var out []node
_, _ = walkPath(n, tail, pathWalk{
choice: func(c *choice, _ []string) ([]node, error) { return c.items, nil },
leaf: func(n node) error { out = append(out, n); return nil },
})
return out
}
// nodeScope is the set of nodes one validation pass covers: a whole loaded tree, func (m renderCounts) renderCount(n node) int {
// or a single inline node.
type nodeScope func(fn func(path string, n node) error) error
func treeScope(root map[string]node) nodeScope {
return func(fn func(path string, n node) error) error { return walkNodes(root, fn) }
}
func inlineScope(n node, label string) nodeScope {
return func(fn func(path string, m node) error) error { return eachNode(n, label, fn) }
}
func checkScope(s nodeScope) error {
if err := checkColumns(s); err != nil {
return err
}
return checkRenders(s)
}
// checkRenders runs the per-node fences over a scope, each over the whole scope
// before the next, so which of several broken nodes is reported does not depend on
// the walk. It runs after checkNoCycles, whose guarantee is what lets the walks
// terminate.
func checkRenders(s nodeScope) error {
mem := reachMemo{}
if err := s(func(path string, n node) error { return repeatCheck(path, n, mem) }); err != nil {
return err
}
if err := s(heldCheck); err != nil {
return err
}
fence := &drawCheck{}
refs := false
if err := s(func(path string, n node) error {
if t, ok := n.(*template); ok && len(t.refs) > 0 {
refs = true
}
return fence.checkDrawGroup(path, n)
}); err != nil {
return err
}
if refs {
if err := s(fence.checkDraws); err != nil {
return err
}
if err := s(fence.checkRecordDraws); err != nil {
return err
}
}
return s((&valueProof{}).checkDatatype)
}
type reachMemo map[node]int
func (m reachMemo) of(n node) int {
if r, done := m[n]; done { if r, done := m[n]; done {
return r return r
} }
r := 1 r := 1
for _, e := range renderEdges(n) { for _, e := range renderEdges(n) {
if c := m.of(e.to); c > r { if c := m.renderCount(e.to); c > r {
r = c r = c
} }
} }
@@ -260,9 +151,10 @@ func (m reachMemo) of(n node) int {
// repeatCheck bounds the renders a repeat multiplies to along any root-to-leaf // repeatCheck bounds the renders a repeat multiplies to along any root-to-leaf
// path, so nested repeats cannot build what one repeat may not. // path, so nested repeats cannot build what one repeat may not.
func repeatCheck(path string, n node, mem reachMemo) error { // docs/decisions.md#the-repeat-cap-bounds-renders-not-bytes
if t, ok := n.(*template); ok && t.repeat > 1 && mem.of(n) > MaxRepeat { func repeatCheck(label string, n node, mem renderCounts) error {
return fmt.Errorf("%s: repeat %d multiplies to %d renders along one path, above the maximum %d", path, t.repeat, mem.of(n), MaxRepeat) if t, ok := n.(*template); ok && t.repeat > 1 && mem.renderCount(n) > MaxRepeat {
return fmt.Errorf("%s: repeat %d multiplies to %d renders along one path, above the maximum %d", label, t.repeat, mem.renderCount(n), MaxRepeat)
} }
return nil return nil
} }
@@ -270,32 +162,42 @@ func repeatCheck(path string, n node, mem reachMemo) error {
// checkNoCycles rejects a reference cycle: a node whose rendering can reach itself // checkNoCycles rejects a reference cycle: a node whose rendering can reach itself
// — directly, mutually, or through a chain — never terminates, so it must fail at // — directly, mutually, or through a chain — never terminates, so it must fail at
// New rather than stack-overflow at render. It is a depth-first walk of the render // New rather than stack-overflow at render. It is a depth-first walk of the render
// graph (renderEdges); grey marks nodes on the current path so a back-edge to one // graph (renderEdges). Every node is a root: a field its parent's format never
// is the cycle, while black lets a shared node (a DAG, not a cycle) be skipped. // renders is still reachable by dot path, so a cycle in one would otherwise reach
// Every node is a root: a field its parent's format never renders is still reachable // render and be fatal there.
// by dot path, so a cycle in one would otherwise reach render and be fatal there. func checkNoCycles(s nodeSet) error {
func checkNoCycles(root map[string]node) error {
const ( const (
grey = 1 grey = 1
black = 2 black = 2
) )
inSet := map[node]bool{}
_ = s(func(_ string, n node) error {
inSet[n] = true
if t, isTable := n.(*table); isTable {
inSet[t.rowNode] = true // a selector lands on it, and contained lists it nowhere
}
return nil
})
color := map[node]int{} color := map[node]int{}
var visit func(n node, path string) error var visit func(n node, label string) error
visit = func(n node, path string) error { visit = func(n node, label string) error {
if !inSet[n] {
return nil // loaded before these nodes and proven then, so it never reaches back into it
}
switch color[n] { switch color[n] {
case grey: case grey:
return fmt.Errorf("reference cycle: %s", path) return fmt.Errorf("reference cycle: %s", label)
case black: case black:
return nil return nil
} }
color[n] = grey color[n] = grey
for _, e := range renderEdges(n) { for _, e := range renderEdges(n) {
if err := visit(e.to, path+" -> "+e.label); err != nil { if err := visit(e.to, label+" -> "+e.label); err != nil {
return err return err
} }
} }
color[n] = black color[n] = black
return nil return nil
} }
return walkNodes(root, func(path string, n node) error { return visit(n, path) }) return s(func(label string, n node) error { return visit(n, label) })
} }
-358
View File
@@ -1,358 +0,0 @@
package fejkdata
import (
"fmt"
"sort"
"strings"
)
// heldCheck rejects every route to a held sibling name except the ones that read its
// draw. An expansion holds one draw of that name; anything else that renders it draws
// again, and the two disagree. checkNoOverlap settles the spellings within one format
// (a token, an operand); this settles the rest — a reference, whether it sits in that
// format or in anything the format renders, however deep. A reference path is held
// for the whole render instead, which drawCheck fences.
func heldCheck(path string, n node) error {
t, ok := n.(*template)
if !ok || len(t.held) == 0 {
return nil
}
readers := boundReaders(t.format, t.bound, t.refs)
for _, head := range heldHeads(t) {
if _, isPath := t.bound[head]; isPath && isRef(head) {
continue
}
if err := checkHeadHeld(t, head, readers); err != nil {
return fmt.Errorf("%s: %w", path, err)
}
}
return nil
}
// heldHeads lists a template's held names, operand heads first, then paths, each
// in name order: a level read both ways is reported by the operand's fence, and
// which overlap is reported does not vary.
func heldHeads(t *template) []string {
heads := make([]string, 0, len(t.held))
for head := range t.held {
heads = append(heads, head)
}
sort.Slice(heads, func(i, j int) bool {
_, pi := t.bound[heads[i]]
_, pj := t.bound[heads[j]]
if pi != pj {
return !pi
}
return heads[i] < heads[j]
})
return heads
}
// pinned collects what the hold of head answers for: a path pins the levels it
// passes through and the leaf it lands on, an operand exactly the value its render
// produces.
func pinned(t *template, head string, readers []reader) map[node]bool {
held := map[node]bool{}
if _, isPath := t.bound[head]; !isPath {
operandDraw(t.fields[head], held)
return held
}
for _, r := range readers {
if a := splitArm(r.name, t.refs); a.key == head {
coverPath(t.fields[head], a.tail, held)
}
}
return held
}
// checkHeadHeld rejects every route to what head's hold pins except the readers
// holding it. One seen set across the edges: a node that cannot reach the level
// cannot reach it by another route either, so it is walked once.
func checkHeadHeld(t *template, head string, readers []reader) error {
held := pinned(t, head, readers)
if len(held) == 0 {
return nil // a fixed head holds nothing to reach
}
reader, isPath := t.bound[head]
seen := map[node]bool{}
for _, e := range renderEdges(t) {
if splitArm(e.label, t.refs).key == head || !renders(e.to, held, seen) {
continue
}
if isPath {
return fmt.Errorf("%s renders %q, which {%s} reads a path into; name the fields you want instead", e.reached(), head, reader)
}
return fmt.Errorf("%s renders %q, which a {%s()} also reads; reach it one way so it is drawn once", e.reached(), head, operandReader(t, head))
}
return nil
}
// operandReader names the builtin whose operand holds head.
func operandReader(t *template, head string) string {
fn := ""
_ = eachToken(t.format, func(tok ftoken) error {
if tok.kind != 'b' || fn != "" {
return nil
}
if name, _, isFunc := funcCall(tok.body); isFunc {
for _, operand := range tokenOperands(tok.body) {
if splitArm(operand, t.refs).key == head {
fn = name
}
}
}
return nil
})
return fn
}
// coverPath collects what holding one path pins: every choice level the path
// passes through, whole, and the leaf it renders.
func coverPath(n node, tail []string, into map[node]bool) {
_, _ = walkPath(n, tail, pathWalk{
choice: func(c *choice, _ []string) ([]node, error) { cover(c, into, false); return nil, nil },
leaf: func(n node) error { cover(n, into, false); return nil },
})
}
// cover collects a level and everything contained in it. A fixed string outside a
// choice is left out — it cannot disagree with itself — but inside one each
// variant carries its own, so there it counts.
func cover(n node, into map[node]bool, inChoice bool) {
if isFixed(n) && !inChoice {
return
}
into[n] = true
_, isChoice := n.(*choice)
for _, c := range contained(n) {
cover(c.node, into, inChoice || isChoice)
}
}
// operandDraw collects what one held draw of an operand answers for: the operand
// and what rendering it settles inside itself. The builtin renders its operand
// whole, so that draw fixes every value the render produced, and a second route to
// any of them disagrees with it.
//
// The walk stops at a reference edge, which is where the operand's own value ends
// and a shared source begins: two names referencing one category are two draws, the
// same rule {word} {word} follows.
func operandDraw(n node, into map[node]bool) {
if isFixed(n) {
return
}
if into[n] {
return
}
into[n] = true
for _, e := range renderEdges(n) {
if isRef(e.label) {
continue
}
operandDraw(e.to, into)
}
}
// isFixed is a string that varies nothing: fixed text with no fields to read into.
func isFixed(n node) bool {
t, ok := n.(*template)
return ok && t.fixed && len(t.fields) == 0
}
// renders reports whether rendering n can reach anything in want, following the
// same edges expand does. seen keeps a node shared by several routes from being
// walked twice; checkNoCycles has already proved the graph is a DAG, so the walk
// ends.
func renders(n node, want, seen map[node]bool) bool {
if want[n] {
return true
}
if seen[n] {
return false
}
seen[n] = true
for _, e := range renderEdges(n) {
if renders(e.to, want, seen) {
return true
}
}
return false
}
// checkNoOverlap rejects a format that both renders a sibling level and reads a path
// into it — {p} beside {p.first}, {p.addr} beside {p.addr.city}. The path reads the
// level's held draw while rendering the level expands it afresh, so their values would
// disagree. Reads are compared in sorted order, so which pair is reported does not
// depend on where the tokens sit.
func checkNoOverlap(format string, bound map[string]string, refs map[string]refBinding) error {
names := boundReaders(format, bound, refs)
// Stable over one format-order scan, so two readers of one name (a token and a
// calc operand both naming "p") are reported as the format writes them.
sort.SliceStable(names, func(i, j int) bool { return names[i].path < names[j].path })
for i, level := range names {
for _, path := range names[i+1:] {
if strings.HasPrefix(path.path, level.path+".") {
return fmt.Errorf("%s renders a level that {%s} reads a path into; name the fields you want instead", level.label, path.name)
}
}
}
return nil
}
// reader is one way a format reaches a bound field: as written, by its one
// spelling, and how to name it.
type reader struct{ name, path, label string }
// boundReaders lists every way a format reaches a bound sibling field, in the order the
// format writes them. An operand renders its field, so it names a level exactly
// as a token does; one scan finds both, which is what puts them in one order.
func boundReaders(format string, bound map[string]string, refs map[string]refBinding) []reader {
var names []reader
_ = eachToken(format, func(t ftoken) error {
if t.kind != 'b' {
return nil
}
if fn, _, isFunc := funcCall(t.body); isFunc {
for _, operand := range tokenOperands(t.body) {
a := splitArm(operand, refs)
if _, isBound := bound[a.key]; isBound && !isRef(a.key) {
names = append(names, reader{a.name, a.path, fmt.Sprintf("%s operand %q", fn, operand)})
}
}
return nil
}
for _, a := range splitArms(t.body, refs) {
if _, isBound := bound[a.key]; isBound && !isRef(a.key) {
names = append(names, reader{a.name, a.path, "token {" + a.name + "}"})
}
}
return nil
})
return names
}
// checkNoRepeatedRead rejects a bare token repeated on a held name: {w} {w} beside
// {uppercase(w)} would read one draw twice, where {w} {w} alone draws twice. The
// error names the single-token spelling.
func checkNoRepeatedRead(format string, c formatOps, refs map[string]refBinding) error {
count := map[string]int{}
return eachToken(format, func(t ftoken) error {
if t.kind != 'b' {
return nil
}
if _, _, isFunc := funcCall(t.body); isFunc {
return nil
}
for _, a := range splitArms(t.body, refs) {
if len(a.tail) > 0 || !c.held[a.key] {
continue
}
if count[a.key]++; count[a.key] > 1 {
return fmt.Errorf("token {%s} is repeated, and %s holds %q to one draw per expansion; write {%s} once", a.name, c.holder[a.key], a.key, a.name)
}
}
return nil
})
}
// draws is what has already been drawn for held names: the variant each was drawn as,
// so every path under it reads one row, and the draw each read made, by its one
// spelling, so the same read written twice reads one draw. An expansion keeps one for
// its sibling names, and a render one per group for its reference paths.
type draws struct {
variant map[string]node
value map[string]draw
pins [8]tablePin // the rows pinned, inline so a render over a country's five-deep geo tree stays off the heap
npins int
more map[*table]int // the rows pinned past the inline eight
s *session // what draws a row; nil where a walk only proves selectors
}
// draw is what one read drew: its text, and whether it landed on a null.
type draw struct {
text string
null bool
}
// readField renders one arm of a token. An arm's key is a sibling field or a
// reference linkRefs bound into fields. A name the expansion holds — a level some
// token addresses by dotted path, or a field an operand reads — is drawn once and
// kept, so {place.postal-code} and {place.locality} read one row, either read twice
// gives one value, and a shown operand is the operand computed. Every other name is
// drawn afresh, so {word} {word} still draws twice. checkTokens, checkPath and
// linkRefs prove every step, so the walk cannot fail.
func readField(s *session, t *template, held *draws, sc drawScope, a arm) draw {
if !t.held[a.key] {
if len(a.tail) > 0 {
panic(fmt.Sprintf("fejkdata: %q reads a path into %q, which the expansion does not hold", a.name, a.key))
}
return draw{text: render(s, t.fields[a.key], sc)}
}
d := readScope(s, held, sc, a)
if r, done := d.value[a.path]; done {
return r
}
leaf, err := walkPath(t.fields[a.key], a.tail, pathWalk{
// Hold the draw at every level passed through, so two paths sharing a
// prefix share it.
choice: func(c *choice, rest []string) ([]node, error) {
key := a.key
if consumed := len(a.tail) - len(rest); consumed > 0 {
key = a.steps[consumed-1]
}
n, drew := d.variant[key]
if !drew {
n = drawn(s, c)
if d.variant == nil {
d.variant = map[string]node{}
}
d.variant[key] = n
}
return []node{n}, nil
},
pins: d,
})
if err != nil {
panic(fmt.Sprintf("fejkdata: %q: %v; a fence should have refused this at New", a.name, err))
}
r := renderLeaf(s, leaf, sc)
if d.value == nil {
d.value = map[string]draw{}
}
d.value[a.path] = r
return r
}
// readScope is the draws a held read keeps its draw in: for a reference that reads a path,
// the render's draws for its group, so its draw spans the render; for a sibling, or a
// reference read whole, held.
func readScope(s *session, held *draws, sc drawScope, a arm) *draws {
if isRef(a.key) && len(a.tail) > 0 {
return sc.draws(s)
}
return held
}
// renderLeaf draws and renders what a read lands on: null on a null item, or on a column of one
// reference alone whose read drew null.
func renderLeaf(s *session, n node, sc drawScope) draw {
n = drawn(s, n)
if _, isNull := n.(*null); isNull {
return draw{null: true}
}
r := draw{text: render(s, n, sc)}
if t, _ := n.(*template); t != nil && t.readsColumn != nil {
r.null = sc.in(t).draws(s).value[t.readsColumn.a.path].null
}
return r
}
// drawn resolves a choice to one variant, so a bound head is a concrete node the
// rest of the expansion shares. Nested choices unwrap too: a draw is one value, not
// another set to pick from.
func drawn(s *session, n node) node {
for c, ok := n.(*choice); ok; c, ok = n.(*choice) {
n = pick(s, c)
}
return n
}
-771
View File
@@ -1,771 +0,0 @@
package fejkdata
import (
"fmt"
"regexp"
"strings"
"testing"
)
// This file pins the one-draw rule for dotted sibling tokens. A format that
// addresses a sibling by path ({place.locality}) draws that sibling once per
// expansion, so every {place.*} token in the same format reads the same variant.
// That is what makes a correlated pair — a locality and a postal code that agree
// — expressible in data, without the row having to be the whole node.
// swedishPlaces is a two-variant sibling whose variants pair a locality with the
// postal-code prefix that really belongs to it.
const swedishPlaces = `{"format":"%s","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`
// agree reports whether a rendered "postcode locality" pair is a real pairing.
var agree = regexp.MustCompile(`^(1[0-9]{2} [0-9]{2} Stockholm|573 [0-9]{2} Tranås)$`)
func places(format string) string {
return strings.Replace(swedishPlaces, "%s", format, 1)
}
func TestDottedTokenCorrelatesSiblingFields(t *testing.T) {
f := engine(1)
seen := map[string]bool{}
for i := 0; i < 500; i++ {
got := mustRender(t, f, places("{place.postal-code} {place.locality}"))
if !agree.MatchString(got) {
t.Fatalf("draw %d = %q, want a postcode and locality that pair", i, got)
}
seen[got[len(got)-4:]] = true
}
if len(seen) < 2 {
t.Fatalf("500 draws produced only %v, want both localities", seen)
}
}
func TestRenderingALevelAndReadingIntoItIsRejected(t *testing.T) {
// One draw, one spelling. A token that renders a level and one that reads a
// path into it name the same draw two ways, and only the path reads what the
// other rendered — so the pair is a load error rather than a shape whose two
// halves can disagree.
rejected := map[string]string{
"bare head beside a path": `{"format":"{p} + {p.first}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
"prefix path beside a path": `{"format":"{p.addr}|{p.addr.city}","p":{"format":"{addr}","addr":[` +
`{"format":"{city}","city":["Kiruna","Boden"]},{"format":"{city}","city":["Malmö","Lund"]}]}}`,
"either order": `{"format":"{p.first} + {p}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
}
for name, file := range rejected {
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file})))
if err == nil {
t.Errorf("%s: New = nil error, want the overlapping tokens rejected", name)
continue
}
if !strings.Contains(err.Error(), "reads a path into") {
t.Errorf("%s: New = %v, want it to name the overlap", name, err)
}
}
// The same path twice is one spelling, so it stays legal.
if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
"cat": `{"format":"{p.first} + {p.first}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
}))); err != nil {
t.Errorf("New = %v, want one path read twice accepted", err)
}
}
func TestCalcOperandNamingABoundLevelIsRejected(t *testing.T) {
// A calc operand renders its field, so naming a bound level in one is the same
// overlap as a bare token: {calc(item * 1)} renders what {item.price} reads a
// path into, and the two disagree.
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
"cat": `{"format":"{item.price}|{calc(item * 1)}","item":[` +
`{"format":"{price}","price":"10"},{"format":"{price}","price":"20"}]}`,
})))
if err == nil || !strings.Contains(err.Error(), "reads a path into") {
t.Fatalf("New = %v, want the calc operand rejected as an overlap", err)
}
// Arithmetic over fields no path names is untouched.
if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
"cat": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`,
}))); err != nil {
t.Errorf("New = %v, want plain arithmetic accepted", err)
}
}
func TestReferenceNamingABoundLevelIsRejected(t *testing.T) {
// A reference can name a bound level from the data root, which renders it
// afresh beside the path that reads its held draw — the same overlap by
// another spelling.
// A category never references itself, so each reference back into cat sits in a
// second category cat renders.
rejected := map[string]map[string]string{
"reference names the head": {
"cat": `{"format":"{p.first}|{/hop}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
"hop": `"{/cat.p}"`,
},
"reference names the leaf": {
"cat": `{"format":"{p.addr}|{/hop}","p":{"format":"x","addr":["A","B","C","D"]}}`,
"hop": `"{/cat.p.addr}"`,
},
// The reference need not sit in the format that binds: any field it renders
// reaches the level just the same, however deep.
"reference from a sibling field": {
"cat": `{"format":"{p.first}|{inner}","p":[` +
`{"format":"{first}-{last}","first":"A","last":"1"},{"format":"{first}-{last}","first":"B","last":"2"}],` +
`"inner":"{/hop}"}`,
"hop": `"{/cat.p}"`,
},
}
for name, files := range rejected {
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, files)))
if err == nil || !strings.Contains(err.Error(), "reads a path into") {
t.Errorf("%s: New = %v, want the reference rejected as an overlap", name, err)
}
}
// A reference to anything this format does not bind is untouched.
if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
"cat": `{"format":"{p.first} {/surname}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
"surname": `["Eriksson","Lindqvist"]`,
}))); err != nil {
t.Errorf("New = %v, want a reference outside the bound level accepted", err)
}
}
func TestCycleReachedOnlyByAPathTokenIsRejected(t *testing.T) {
// A path token renders what it lands on, not the level it started from, so the
// cycle walk has to follow it there. A head whose own format names nothing
// would otherwise hide the cycle until render, where it is fatal.
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
"a": `{"format":"{p.x}","p":{"format":"static","x":"{/b}"}}`,
"b": `"{/a}"`,
})))
if err == nil || !strings.Contains(err.Error(), "reference cycle") {
t.Fatalf("New = %v, want the cycle through {p.x} rejected", err)
}
}
func TestALevelRenderedOnlyByAPathTokenIsHeld(t *testing.T) {
// {p.a} renders q, so it is a route to the level {q.x} holds — even though p's
// own format names nothing.
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
"thing": `{"format":"{p.a} {q.x}","p":{"format":"static","a":"{/hop}"},` +
`"q":{"format":"{x}","x":["1","2"]}}`,
"hop": `"{/thing.q}"`,
})))
if err == nil || !strings.Contains(err.Error(), "reads a path into") {
t.Fatalf("New = %v, want the second route to q rejected", err)
}
}
// TestACalcOperandIsHeldAgainstEveryRoute pins what a {calc()} operand's hold
// fences. A calc renders its operand whole, so the draw it holds is that one node's
// value: another route conflicts only by naming that same node, and then the value
// shown is not the value computed. Two names that merely draw from one source are
// two draws, as {word} {word} is — the rule everywhere else in the engine.
func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) {
rejected := map[string]struct {
files map[string]string
want string // the route the error names, spelled as the author wrote it
}{
// A category never references itself, so each route back into cat sits in a
// second category cat renders.
"a reference beside the operand": {
map[string]string{
"cat": `{"format":"{/hop} x 2 = {calc(net * 2, 2)}","net":["10.00","20.00"]}`,
"hop": `"{/cat.net}"`,
},
`{/hop} renders "net"`,
},
"a reference one level down": {
map[string]string{
"cat": `{"format":"{calc(net * 2, 2)} {q}","net":["10.00","20.00"],` +
`"q":"{/hop}"}`,
"hop": `"{/cat.net}"`,
},
`{q} renders "net"`,
},
// A reference ending at the operand binds the choice itself, not a variant,
// so wrapping the operand in one changes nothing.
"a reference to an operand wrapped in a choice": {
map[string]string{
"cat": `{"format":"{calc(n * 2, 2)} {q}","n":{"format":"{v}","v":["1","2"]},` +
`"q":"{/hop}"}`,
"hop": `"{/cat.n}"`,
},
`{q} renders "n"`,
},
// The reaching route can be another operand, which is not a token — so the
// error must name it as one rather than invent a {b} the format never wrote.
"an operand reaching another operand": {
map[string]string{
"cat": `{"format":"{calc(a + b, 0)}","a":{"format":"{x}","x":["1","2"]},` +
`"b":"{/hop}"}`,
"hop": `"{/cat.a}"`,
},
`calc operand "b" renders "a"`,
},
// Reaching *into* the operand is the same shape as naming it: the draw the
// calc holds settled that value too. The token spelling {net.v} is already
// rejected, so the reference spelling has to be.
"a reference into the operand": {
map[string]string{
"cat": `{"format":"{calc(net * 2, 2)}|{/hop}",` +
`"net":{"format":"{v}","v":["10.00","20.00"]}}`,
"hop": `"{/cat.net.v}"`,
},
`{/hop} renders "net"`,
},
"a reference into the operand one level down": {
map[string]string{
"cat": `{"format":"{calc(net * 2, 2)}|{q}","net":{"format":"{v}","v":["10.00","20.00"]},` +
`"q":"{/hop}"}`,
"hop": `"{/cat.net.v}"`,
},
`{q} renders "net"`,
},
// Each head is walked against its own seen set. Sharing one across heads
// would let the first head's walk mark the only route to the second's draw,
// and this violation sits on the later head in sorted order.
"a violation on the second of two held heads": {
map[string]string{
"cat": `{"format":"{calc(a + b, 0)} {w}","a":{"format":"{x}","x":["1","2"]},` +
`"b":{"format":"{y}","y":["3","4"]},"w":"{/hop}"}`,
"hop": `"{/cat.b}"`,
},
`{w} renders "b"`,
},
}
for name, c := range rejected {
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files)))
if err == nil || !strings.Contains(err.Error(), "a {calc()} also reads") {
t.Errorf("%s: New = %v, want the second route to the operand rejected", name, err)
continue
}
if !strings.Contains(err.Error(), c.want) {
t.Errorf("%s: New = %v, want it to name %q", name, err, c.want)
}
}
accepted := map[string]map[string]string{
// The spellings that read the one draw: the operand itself, and a bare token.
"only the operand and a bare token": {
"cat": `{"format":"{net} x 2 = {calc(net * 2, 2)}","net":["10.00","20.00"]}`,
},
// A sibling the operand's format never renders is no part of its value. A
// path head differs — a path may read into anything the level contains —
// which is why that half of the fence covers containment.
"a reference to a sibling the operand never renders": {
"cat": `{"format":"{calc(net * 2, 2)} {unit}",` +
`"net":{"format":"{v}","v":["1","2"],"spare":["kg","lb"]},` +
`"unit":"{/hop}"}`,
"hop": `"{/cat.net.spare}"`,
},
// Two operands drawing from one source are two names, so two draws: each is
// held under its own name and shown once, and neither can disagree.
"two operands sharing one source": {
"die": `["1","2","3","4","5","6"]`,
"cat": `{"format":"{d1} + {d2} = {calc(d1 + d2, 0)}","d1":"{/die}",` +
`"d2":"{/die}"}`,
},
// Likewise a reference drawing from what the operand draws from: {/common}
// and net are two names, not two spellings of one field.
"a reference to what an operand renders through": {
"common": `["1","2"]`,
"cat": `{"format":"{calc(net * 2, 2)} {/common}","net":"{/common}"}`,
},
// A fixed string cannot disagree with itself, so it needs no fence.
"a literal operand named twice": {
"cat": `{"format":"{calc(n * 2, 0)} {/hop}","n":"5"}`,
"hop": `"{/cat.n}"`,
},
// One node reached twice while walking the operand: the walk must not
// revisit it, and the repeat is not a second route to anything.
"an operand that renders one field twice": {
"cat": `{"format":"{calc(v * 2, 0)}","v":{"format":"{a}{a}","a":["1","2"]}}`,
},
}
for name, files := range accepted {
if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil {
t.Errorf("%s: New = %v, want it accepted", name, err)
}
}
}
func TestAPathReachesEveryVariantItMightDraw(t *testing.T) {
// A path through a choice may land in any variant, so both walks have to see
// all of them. Reaching only the first leaves whatever hides in a later
// variant to be found at render — a cycle there is fatal, and a second route
// to a held level disagrees silently.
rejected := map[string]struct {
files map[string]string
want string
}{
"a cycle in a later variant": {
map[string]string{
"cat": `{"format":"{p.x}","p":[{"format":"h","x":"safe"},{"format":"h","x":"{/hop}"}]}`,
"hop": `"{/cat}"`,
},
"reference cycle",
},
"a second route in a later variant": {
map[string]string{
"cat": `{"format":"{p.x} {q.y}","p":[{"format":"h","x":"safe"},{"format":"h","x":"{/hop}"}],` +
`"q":{"format":"{y}","y":["1","2"]}}`,
"hop": `"{/cat.q}"`,
},
"reads a path into",
},
}
for name, c := range rejected {
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files)))
if err == nil || !strings.Contains(err.Error(), c.want) {
t.Errorf("%s: New = %v, want it to mention %q", name, err, c.want)
}
}
}
func TestALevelAPathNeverRendersIsAccepted(t *testing.T) {
// A path token does not expand its head's format, so a reference sitting in
// that format is not a second route to anything: it is never rendered by the
// path at all. Both orders must load. p and q sit one level down, so they are
// one column of thing rather than two that would read one another.
accepted := map[string]map[string]string{
"reference in the head's own format": {
"thing": `{"format":"{inner}","inner":{"format":"{p.first} {q.a}",` +
`"p":{"format":"{first} {/hop}","first":["A","B"]},"q":{"format":"{a}","a":["1","2"]}}}`,
"hop": `"{/thing.inner.q}"`,
},
"the mirror shape": {
"thing": `{"format":"{inner}","inner":{"format":"{p.first} {q.a}",` +
`"p":{"format":"{first}","first":["A","B"]},"q":{"format":"{a} {/hop}","a":["1","2"]}}}`,
"hop": `"{/thing.inner.p}"`,
},
}
for name, files := range accepted {
if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil {
t.Errorf("%s: New = %v, want it accepted", name, err)
}
}
}
func TestADeepDiamondChainLoads(t *testing.T) {
// A diamond chain is 2^n routes through n nodes. The search past a bound level
// must walk each node once, not once per route, or a deep chain never loads.
files := map[string]string{"l0": `"x"`}
for i := 1; i <= 30; i++ {
files[fmt.Sprintf("l%d", i)] = fmt.Sprintf(
`{"format":"{a}{b}","a":"{/l%d}","b":"{/l%d}"}`, i-1, i-1)
}
files["thing"] = `{"format":"{p.first} {/l30}","p":{"format":"x","first":["A","B"]}}`
if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil {
t.Fatalf("New = %v, want a deep diamond chain to load", err)
}
}
func TestASharedNodeIsWalkedOnce(t *testing.T) {
// Two fields reaching one node make the render graph a diamond, not a tree.
// The search past a bound level must take that in its stride rather than walk
// the shared node once per route.
f, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
"cat": `{"format":"{p.first}|{q}","p":{"format":"{first}","first":["Anna","Bo"]},` +
`"q":{"format":"{a}{b}","a":"{/shared}","b":"{/shared}"}}`,
"shared": `"x"`,
})), WithSeed(1))
if err != nil {
t.Fatalf("New = %v, want a shared node accepted", err)
}
if got, err := f.Fake("cat"); err != nil || !strings.HasSuffix(got, "|xx") {
t.Fatalf("Fake(cat) = %q, %v, want it to end in xx", got, err)
}
}
func TestTheEarlierReaderIsNamed(t *testing.T) {
// A token and a calc operand can name one level. The one the format writes
// first is the one reported, so the error points at the same place a reader
// looking at the format would start.
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
"cat": `{"format":"{calc(p * 1)} {p} {p.first}","p":{"format":"{first}","first":["1","2"]}}`,
})))
if err == nil || !strings.Contains(err.Error(), `calc operand "p"`) {
t.Fatalf("New = %v, want the calc operand named, being written first", err)
}
}
func TestReferenceToAMatchingStringIsAccepted(t *testing.T) {
// A literal renders one fixed string, so no draw of it can disagree with a
// held one. Two unrelated literals that merely spell the same text must not
// read as the same node — the trap when comparing a value type.
if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
"cat": `{"format":"{p.city} {/other.tag}","p":{"format":"{city}","city":"Stockholm"}}`,
"other": `{"format":"x","tag":"Stockholm"}`,
}))); err != nil {
t.Fatalf("New = %v, want a reference to a matching string accepted", err)
}
}
func TestNestedChoiceDrawsOneVariant(t *testing.T) {
// A choice item may itself be a choice, so drawing a bound head unwraps until
// it reaches a value. Stopping at one level leaves a choice where the walk
// expects a template.
f := engine(21)
seen := map[string]bool{}
for i := 0; i < 200; i++ {
seen[mustRender(t, f, `{"format":"{p.x}","p":[{"format":"{x}","x":"1"},{"format":"{x}","x":"2"}]}`)] = true
}
if !seen["1"] || !seen["2"] || len(seen) != 2 {
t.Fatalf("200 draws produced %v, want 1 and 2", seen)
}
}
func TestRepeatingLevelIsNamedInTheError(t *testing.T) {
// The level carrying the repeat is the one to fix, so the error names it
// rather than the head the path started from.
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
"cat": `{"format":"[{p.a.b}]","p":{"format":"{a}","a":{"format":"{b}","repeat":3,"separator":",","b":"z"}}}`,
})))
if err == nil || !strings.Contains(err.Error(), `"p.a"`) {
t.Fatalf("New = %v, want it to name the level p.a that carries the repeat", err)
}
}
func TestPathIntoARepeatingLevelBehindAChoiceIsRejected(t *testing.T) {
// A choice of rows is the shape this feature is for, so the repeat rule has to
// reach inside one — otherwise the direct spelling is a load error and the same
// mistake behind a choice silently drops the repeat.
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
"cat": `{"format":"[{p.a}]","p":[{"format":"{a}","repeat":3,"separator":",","a":"x"},{"format":"{a}","a":"y"}]}`,
})))
if err == nil || !strings.Contains(err.Error(), "repeat") {
t.Fatalf("New = %v, want the repeat behind a choice rejected", err)
}
}
func TestPathIntoARepeatingLevelIsRejected(t *testing.T) {
// A path reads one level's draw, so it can never apply that level's repeat.
// The engine rejects options that cannot take effect, and this is one.
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
"cat": `{"format":"[{p.a}]","p":{"format":"{a}","repeat":3,"separator":",","a":"z"}}`,
})))
if err == nil || !strings.Contains(err.Error(), "repeat") {
t.Fatalf("New = %v, want a path into a repeating level rejected", err)
}
_, err = New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
"cat": `{"format":"[{p.a}]","p":{"format":"{a}","a":"{/word.w}","drawGroup":"g"}}`,
"word": `{"format":"{w}","w":["x","y"]}`,
})))
if err == nil || !strings.Contains(err.Error(), `the level "p" carries a drawGroup`) {
t.Fatalf("New = %v, want a path into a level carrying a group rejected", err)
}
}
func TestPathIntoAPlainTemplateNamesTheMissingField(t *testing.T) {
// A head that is one template, not a choice, reports the missing segment
// directly — there are no variants to compare.
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
"cat": `{"format":"{a.nope}","a":{"format":"x","b":"1"}}`,
})))
if err == nil || !strings.Contains(err.Error(), `no field "nope"`) {
t.Fatalf("New = %v, want it to name the missing field", err)
}
}
func TestFieldWithoutAPathTokenStillDrawsEachTime(t *testing.T) {
// A format with no dotted token keeps today's behaviour: every {word} is an
// independent draw.
f := engine(3)
varied := false
for i := 0; i < 200 && !varied; i++ {
got := mustRender(t, f, `{"format":"{w}{w}{w}","w":["a","b","c","d","e"]}`)
varied = got[0] != got[1] || got[1] != got[2]
}
if !varied {
t.Fatal("200 draws of {w}{w}{w} never varied, want independent draws")
}
}
func TestTwoBoundHeadsDrawIndependently(t *testing.T) {
// Binding is per head, so two correlated sets in one format compose instead
// of needing a cross product of variants.
f := engine(4)
seen := map[string]bool{}
for i := 0; i < 400; i++ {
seen[mustRender(t, f, `{"format":"{p.v}{q.v}",
"p":[{"format":"{v}","v":"A"},{"format":"{v}","v":"B"}],
"q":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}]}`)] = true
}
for _, want := range []string{"A1", "A2", "B1", "B2"} {
if !seen[want] {
t.Fatalf("400 draws produced %v, want every combination including %q", seen, want)
}
}
}
func TestRepeatRedrawsTheBinding(t *testing.T) {
// The binding lives for one expansion, so each repetition is a fresh draw —
// and each repetition is internally consistent.
f := engine(5)
tmpl := strings.Replace(places("{place.postal-code} {place.locality}"), `"format":"{place.postal-code} {place.locality}"`,
`"format":"{place.postal-code} {place.locality}","repeat":6,"separator":"\n"`, 1)
varied := false
for i := 0; i < 40; i++ {
got := mustRender(t, f, tmpl)
lines := strings.Split(got, "\n")
if len(lines) != 6 {
t.Fatalf("repeat 6 produced %d lines", len(lines))
}
for _, line := range lines {
if !agree.MatchString(line) {
t.Fatalf("repetition %q does not pair", line)
}
}
if lines[0] != lines[1] || lines[1] != lines[2] {
varied = true
}
}
if !varied {
t.Fatal("40 renders of repeat 6 never varied within a render, want a redraw per repetition")
}
}
func TestNestedTemplateKeepsItsOwnBinding(t *testing.T) {
// A binding belongs to the expansion that made it: an inner template's own
// place is not the outer one's.
f := engine(6)
tmpl := `{"format":"{place.v}{inner}",
"place":[{"format":"{v}","v":"A"},{"format":"{v}","v":"B"}],
"inner":{"format":"{place.v}","place":[{"format":"{v}","v":"A"},{"format":"{v}","v":"B"}]}}`
seen := map[string]bool{}
for i := 0; i < 400; i++ {
seen[mustRender(t, f, tmpl)] = true
}
if !seen["AB"] && !seen["BA"] {
t.Fatalf("400 draws produced %v, want the inner binding to be independent", seen)
}
}
func TestBoundDrawIsSeedStable(t *testing.T) {
tmpl := places("{place.postal-code} {place.locality}")
a := mustRender(t, engine(42), tmpl)
b := mustRender(t, engine(42), tmpl)
if a != b {
t.Fatalf("same seed gave %q and %q, want an identical draw", a, b)
}
}
func TestDottedTokenErrors(t *testing.T) {
// A path token is validated where every other token is: at New, naming what
// is wrong, so a typo can never reach a render.
rejected := map[string]struct {
file string
want string
}{
"unknown head": {
`{"format":"{nope.x}","place":{"format":"{v}","v":"A"}}`,
`no field "nope"`,
},
"unknown tail": {
`{"format":"{place.nope}","place":[{"format":"{v}","v":"A"},{"format":"{v}","v":"B"}]}`,
`"nope"`,
},
"tail only some variants carry": {
`{"format":"{place.locality}","place":[{"format":"{locality}","locality":"A"},"x"]}`,
`locality`,
},
"path into a literal": {
`{"format":"{x.y}","x":"plain"}`,
`"y"`,
},
"dotted field key": {
`{"format":"[{a.b}]","a.b":"V"}`,
`contains "."`,
},
}
for name, c := range rejected {
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": c.file})))
if err == nil {
t.Errorf("%s: New = nil error, want it rejected at load", name)
continue
}
if !strings.Contains(err.Error(), c.want) {
t.Errorf("%s: New = %v, want it to mention %q", name, err, c.want)
}
}
}
func TestSamePathReadTwiceReadsOneValue(t *testing.T) {
// A path is drawn once per expansion, so reading it twice reads one value —
// what a name shown in a display form and again in an address needs.
f := engine(7)
for i := 0; i < 300; i++ {
got := mustRender(t, f, `{"format":"{p.first}|{p.first}","p":{"format":"{first}","first":["Anna","Astrid","Elin","Karin"]}}`)
parts := strings.Split(got, "|")
if parts[0] != parts[1] {
t.Fatalf("draw %d = %q, want one value read twice", i, got)
}
}
}
func TestDifferentTailsUnderOneHeadShareTheRow(t *testing.T) {
// Two tails of one head stay in the same variant, each keeping its own value.
f := engine(8)
for i := 0; i < 300; i++ {
got := mustRender(t, f, `{"format":"{p.a}{p.b}",
"p":[{"format":"x","a":"A","b":"1"},{"format":"y","a":"B","b":"2"}]}`)
if got != "A1" && got != "B2" {
t.Fatalf("draw %d = %q, want a row's own pair", i, got)
}
}
}
// deepPlaces nests the correlated facts a level down: the row holds an addr, and
// the addr holds the city and the zip that belongs to it.
const deepPlaces = `{"format":"%s","p":{"format":"{addr}","addr":[{"format":"{city}","city":"Stockholm","zip":"11111"},{"format":"{city}","city":"Kiruna","zip":"98100"}]}}`
func TestPathHoldsItsDrawAtEveryLevel(t *testing.T) {
// A choice below the head is drawn once too, so two paths sharing a prefix
// share every level of it — not just the row they started from.
f := engine(11)
seen := map[string]bool{}
for i := 0; i < 500; i++ {
got := mustRender(t, f, strings.Replace(deepPlaces, "%s", "{p.addr.city}/{p.addr.zip}", 1))
if got != "Stockholm/11111" && got != "Kiruna/98100" {
t.Fatalf("draw %d = %q, want a city and the zip that belongs to it", i, got)
}
seen[got] = true
}
if len(seen) < 2 {
t.Fatalf("500 draws produced only %v, want both", seen)
}
}
func TestPathHoldsItsDrawThreeLevelsDown(t *testing.T) {
// The rule does not run out at two: every level a path passes through is held.
f := engine(12)
tmpl := `{"format":"{p.geo.town.name}/{p.geo.town.zip} {p.geo.region}","p":{"format":"{geo}","geo":[{"format":"{town}","region":"Norrbotten","town":[{"format":"{name}","name":"Kiruna","zip":"98100"},{"format":"{name}","name":"Luleå","zip":"97200"}]},{"format":"{town}","region":"Skåne","town":[{"format":"{name}","name":"Malmö","zip":"21100"},{"format":"{name}","name":"Lund","zip":"22100"}]}]}}`
want := map[string]bool{
"Kiruna/98100 Norrbotten": true, "Luleå/97200 Norrbotten": true,
"Malmö/21100 Skåne": true, "Lund/22100 Skåne": true,
}
seen := map[string]bool{}
for i := 0; i < 600; i++ {
got := mustRender(t, f, tmpl)
if !want[got] {
t.Fatalf("draw %d = %q, want a town with its own zip and region", i, got)
}
seen[got] = true
}
if len(seen) != len(want) {
t.Fatalf("600 draws produced %v, want all four towns", seen)
}
}
func TestPathWalksVariantsOfDifferentShape(t *testing.T) {
// One variant holds addr as a plain template where another holds a choice.
// Both carry the path, so both must walk — the walk reads what it drew, never
// assuming a shape.
f := engine(13)
tmpl := `{"format":"{p.addr.city}/{p.addr.zip}","p":[
{"format":"{addr}","addr":{"format":"{city}","city":"Kiruna","zip":"98100"}},
{"format":"{addr}","addr":[
{"format":"{city}","city":"Malmö","zip":"21100"},
{"format":"{city}","city":"Lund","zip":"22100"}]}]}`
for i := 0; i < 400; i++ {
got := mustRender(t, f, tmpl)
if got != "Kiruna/98100" && got != "Malmö/21100" && got != "Lund/22100" {
t.Fatalf("draw %d = %q, want a city with its own zip", i, got)
}
}
}
func TestDeepPathsUnderOneHeadStayIndependentWhereTheyDiverge(t *testing.T) {
// Two paths share only what they actually share: a common prefix is one draw,
// and each keeps its own draw past the point they part.
f := engine(14)
tmpl := `{"format":"{p.a.v}{p.b.v}","p":{"format":"x","a":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}],"b":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}]}}`
seen := map[string]bool{}
for i := 0; i < 400; i++ {
seen[mustRender(t, f, tmpl)] = true
}
for _, want := range []string{"11", "12", "21", "22"} {
if !seen[want] {
t.Fatalf("400 draws produced %v, want every combination including %q", seen, want)
}
}
}
func TestEmptyPathSegmentIsRejected(t *testing.T) {
// "{a.}" and "{a..b}" are unfinished paths, and the segment naming nothing is
// reported as that rather than as a missing field. An empty name is rejected
// where it is authored, so no data can make these resolve.
rejected := map[string]string{
"trailing dot": `{"format":"[{a.}]","a":"x"}`,
"double dot": `{"format":"[{a..b}]","a":"x"}`,
}
for name, file := range rejected {
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file})))
if err == nil {
t.Errorf("%s: New = nil error, want the unfinished path rejected", name)
continue
}
if !strings.Contains(err.Error(), "empty segment") {
t.Errorf("%s: New = %v, want it to name the empty segment", name, err)
}
}
}
func TestCycleThroughAPathTokenIsRejected(t *testing.T) {
// A path token is a render edge like any other, so a cycle routed through one
// must be caught at New. Reaching render would be fatal: the recursion never
// terminates, and a stack overflow cannot be recovered.
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
"a": `{"format":"{p.x}","p":{"format":"{x}","x":"{/b}"}}`,
"b": `"{/a}"`,
})))
if err == nil || !strings.Contains(err.Error(), "reference cycle") {
t.Fatalf("New = %v, want the cycle through {p.x} rejected", err)
}
}
func TestBoundPathIsReachableByFake(t *testing.T) {
// Binding changes how a format reads a sibling, not what List and Fake offer:
// the sub-fields stay addressable on their own.
dir := writeData(t, map[string]string{"address": places("{place.postal-code} {place.locality}")})
f, err := New(WithoutShippedData(), WithDataPath(dir), WithSeed(9))
if err != nil {
t.Fatalf("New = %v", err)
}
for _, path := range []string{"address", "address.place", "address.place.locality", "address.place.postal-code"} {
if _, err := f.Fake(path); err != nil {
t.Errorf("Fake(%q) = %v, want it to render", path, err)
}
}
}
func TestRepeatedBareTokenOfAHeldNameIsRejected(t *testing.T) {
for src, want := range map[string]string{
`{"format":"{w} {w} {uppercase(w)}","w":["a","b"]}`: "write {w} once",
`{"format":"{w} {w|x} {uppercase(w)}","w":["a","b"],"x":["c","d"]}`: "write {w} once",
`{"format":"{n} + {n} = {calc(n * 2)}","n":["1","2"]}`: "write {n} once",
`{"format":"{p.a} {q} {q} {lowercase(q)}","p":{"format":"{a}","a":"1"},"q":["A","B"]}`: "write {q} once",
} {
_, err := compile(parse(t, src))
if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "holds") {
t.Errorf("compile(%s) = %v, want the repeated token rejected naming %s", src, err, want)
}
}
for _, ok := range []string{
`{"format":"{w} {w}","w":["a","b"]}`,
`{"format":"{w} {uppercase(w)}","w":["a","b"]}`,
`{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00"],"qty":["3","7"]}`,
`{"format":"{uppercase(w)} {uppercase(w)}","w":["a","b"]}`,
} {
if _, err := compile(parse(t, ok)); err != nil {
t.Errorf("compile(%s) = %v, want it accepted", ok, err)
}
}
}
func TestReadFieldPanicsOnAnUnheldPath(t *testing.T) {
tm, ok := compiled(t, `{"format":"{w}","w":{"format":"{x}","x":"1"}}`).(*template)
if !ok {
t.Fatal("not a template")
}
mustPanic(t, "unheld arm with a path", func() {
readField(engine(1).rand, tm, nil, drawScope{}, arm{name: "w.x", key: "w", tail: []string{"x"}, path: "w.x"})
})
}
+120
View File
@@ -0,0 +1,120 @@
package fejkdata
import (
"errors"
"go/build"
"io/fs"
"os"
"path/filepath"
"slices"
"strings"
"testing"
)
// TestPackageImports holds each package in the module to the module packages it may import.
func TestPackageImports(t *testing.T) {
mod, err := os.ReadFile("go.mod")
if err != nil {
t.Fatal(err)
}
module := ""
for _, line := range strings.Split(string(mod), "\n") {
if f := strings.Fields(line); len(f) > 1 && f[0] == "module" {
module = strings.Trim(f[1], `"`)
break
}
}
if module == "" {
t.Fatal("go.mod names no module")
}
checked := 0
allowed := map[string][]string{
".": {"internal/builtinfunc", "internal/datafiles", "internal/datatype", "internal/drawstate", "internal/grammar", "internal/invariant", "internal/jsonvalue", "internal/proven", "internal/rows"},
"cli": {"."},
"cmd/fejkdata": {"cli", "data"},
"data": {"data/en_US", "data/geo/SE", "data/geo/US", "data/misc", "data/sv_SE"},
"data/en_US": {"data/geo/US"},
"data/geo/SE": nil,
"data/geo/US": nil,
"data/misc": nil,
"data/sv_SE": {"data/geo/SE"},
"internal/builtinfunc": {"internal/datatype", "internal/drawstate", "internal/grammar", "internal/invariant", "internal/proven"},
"internal/datafiles": {"internal/grammar", "internal/jsonvalue"},
"internal/datatype": nil,
"internal/drawstate": nil,
"internal/grammar": nil,
"internal/invariant": nil,
"internal/jsonvalue": nil,
"internal/proven": {"internal/datatype", "internal/grammar", "internal/invariant"},
"internal/rows": {"internal/drawstate", "internal/grammar", "internal/invariant"},
}
testsAlso := map[string][]string{
".": {"data", "data/en_US", "data/geo/SE", "data/geo/US", "data/misc", "data/sv_SE"},
"cli": {"internal/grammar"},
"data/en_US": {"."},
"data/geo/SE": {"."},
"data/geo/US": {"."},
"data/misc": {"."},
"data/sv_SE": {"."},
}
seen := map[string]bool{}
eachPackage(t, func(dir string, pkg *build.Package) {
seen[dir] = true
may, listed := allowed[dir]
if !listed {
t.Errorf("package %s is not listed: add the module packages it may import", dir)
return
}
for _, imp := range slices.Concat(pkg.Imports, pkg.TestImports, pkg.XTestImports) {
may := may
if !slices.Contains(pkg.Imports, imp) {
may = slices.Concat(may, testsAlso[dir])
}
if imp != module && !strings.HasPrefix(imp, module+"/") {
continue
}
checked++
rel := strings.TrimPrefix(strings.TrimPrefix(imp, module), "/")
if rel == "" {
rel = "."
}
if rel != dir && !slices.Contains(may, rel) {
t.Errorf("package %s imports %s, which its list does not allow", dir, rel)
}
}
})
if checked == 0 {
t.Fatalf("no package imports a package of module %s, which go.mod names", module)
}
for dir := range allowed {
if !seen[dir] {
t.Errorf("listed package %s does not exist", dir)
}
}
}
// eachPackage calls fn with every Go package in the module and its slash-separated directory.
func eachPackage(t *testing.T, fn func(dir string, pkg *build.Package)) {
t.Helper()
err := filepath.WalkDir(".", func(dir string, d fs.DirEntry, err error) error {
if err != nil || !d.IsDir() {
return err
}
if dir != "." && (strings.HasPrefix(d.Name(), ".") || d.Name() == "testdata") {
return filepath.SkipDir
}
pkg, err := build.ImportDir(dir, 0)
var noGo *build.NoGoError
if errors.As(err, &noGo) {
return nil
}
if err != nil {
return err
}
fn(filepath.ToSlash(dir), pkg)
return nil
})
if err != nil {
t.Fatal(err)
}
}
+188
View File
@@ -0,0 +1,188 @@
package fejkdata
import (
"errors"
"fmt"
"io/fs"
"slices"
"strings"
"github.com/larvit/fejkdata/internal/datafiles"
"github.com/larvit/fejkdata/internal/grammar"
)
// indexed is a category an index names and no call has reached yet, and the source holding it.
type indexed struct {
entry datafiles.IndexEntry
src datafiles.Source
}
// placeIndex sets every category the index names as unloaded in root, in place of what
// root holds at its path.
func placeIndex(root *folder, src datafiles.Source, index map[string]datafiles.IndexEntry) {
for _, p := range sortedNames(index) {
segs := strings.Split(p, ".")
madeFolder(root, segs[:len(segs)-1]).putUnloaded(segs[len(segs)-1], &indexed{entry: index[p], src: src})
}
}
// categoryAt is where a category, loaded or not, sits: the folder holding it, that
// folder's path, and its name.
type categoryAt struct {
dir []string
in *folder
name string
}
func (c categoryAt) unloaded() bool { return c.in.unloaded[c.name] != nil }
// categoryOn is the category, loaded or not, that a path from root names or passes.
func categoryOn(root *folder, segs []string) (categoryAt, bool) {
g, i := folderAt(root, segs)
if i == len(segs) {
return categoryAt{}, false
}
c := categoryAt{dir: segs[:i:i], in: g, name: segs[i]}
return c, c.unloaded() || g.children[c.name] != nil
}
// loadCallerPath strips the / a caller's path may start with, splits the path, and loads the
// unloaded category it names or passes. The caller walks it next, and a walk loads nothing.
func (f *Generator) loadCallerPath(path string) (string, []string, error) {
path, err := grammar.CallerPath(path)
if err != nil {
return "", nil, fmt.Errorf("fejkdata: %w", err)
}
segs, err := grammar.SplitPath(path)
if err != nil {
return "", nil, fmt.Errorf("fejkdata: %w", err)
}
if c, ok := categoryOn(&f.root, segs); ok && c.unloaded() {
if err := loadReached(&f.root, nil, []categoryAt{c}); err != nil {
return "", nil, fmt.Errorf("fejkdata: %w", loadError{err})
}
}
return path, segs, nil
}
// referenced is every category, loaded or not, that the templates of nodes reference, each
// reference read from the folder dir.
func referenced(root *folder, dir []string, nodes nodeSet) []categoryAt {
var out []categoryAt
_ = nodes(func(_ string, n node) error {
t, isTemplate := n.(*template)
if !isTemplate {
return nil
}
for _, name := range refTokens(t.tokens) {
segs, err := refSegments(name, dir)
if err != nil {
continue // resolveRefs reports it
}
if c, ok := categoryOn(root, segs); ok {
out = append(out, c)
}
}
return nil
})
return out
}
// needs is every category, loaded or not, that s references, the table beside s that its
// parent column names, and every unloaded table beside s whose parent s is.
func needs(root *folder, s categorySite) []categoryAt {
out := referenced(root, s.dir, siteNodes([]categorySite{s}))
if parent := parentOf(s.n); parent != "" {
out = append(out, categoryAt{dir: s.dir, in: s.in, name: parent})
}
for _, name := range sortedNames(s.in.unloaded) {
if s.in.unloaded[name].entry.Parent == s.name {
out = append(out, categoryAt{dir: s.dir, in: s.in, name: name})
}
}
return out
}
// loadReached loads the unloaded categories among wanted and every unloaded category they
// or sites need, then runs sites and all it loaded through a whole load's pipeline. On an
// error it puts back what it loaded, so the next call fails the same way.
func loadReached(root *folder, sites []categorySite, wanted []categoryAt) error {
queue := wanted
for _, s := range sites {
queue = append(queue, needs(root, s)...)
}
var b batch
err := b.load(root, queue)
if err == nil {
err = b.run(root, sites)
}
if err != nil {
b.putBack()
}
return err
}
// batch is the categories one loadReached loaded, each beside the entry it was loaded from.
type batch []loaded
type loaded struct {
site categorySite
was *indexed
}
// load loads each unloaded category in queue and what it needs. It stops at a table whose
// parent column disagrees with its entry. Otherwise a table could load without its parent,
// or an earlier table could link to one that putBack then removes.
func (b *batch) load(root *folder, queue []categoryAt) error {
for ; len(queue) > 0; queue = queue[1:] {
c := queue[0]
x := c.in.unloaded[c.name]
if x == nil {
continue
}
if err := x.src.Load(c.dir, c.name, compileInto(func([]string) *folder { return c.in })); err != nil {
if errors.Is(err, fs.ErrNotExist) {
err = fmt.Errorf("%s indexes %s: %w", x.src.ManifestPath(), categoryPath(c.dir, c.name), err)
}
return err
}
s := siteIn(c.dir, c.in, c.name)
*b = append(*b, loaded{site: s, was: x})
if parent := parentOf(s.n); parent != x.entry.Parent {
return fmt.Errorf("%s: the index entry for %s %s, and its table %s", x.src.ManifestPath(), s.path, namesParent(x.entry.Parent), namesParent(parent))
}
queue = append(queue, needs(root, s)...)
}
return nil
}
func (b batch) run(root *folder, sites []categorySite) error {
sites = slices.Clip(sites)
for _, l := range b {
sites = append(sites, l.site)
}
if len(sites) == 0 {
return nil
}
return categoryPipeline(sites, root.children).run()
}
func (b batch) putBack() {
for _, l := range b {
l.site.in.putUnloaded(l.site.name, l.was)
}
}
func namesParent(parent string) string {
if parent == "" {
return "names no parent"
}
return fmt.Sprintf("names parent %q", parent)
}
func parentOf(n node) string {
if t, isTable := n.(*table); isTable {
return t.rows.Options().Parent
}
return ""
}
+52 -105
View File
@@ -3,28 +3,30 @@ package fejkdata
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"strings"
"github.com/larvit/fejkdata/internal/grammar"
"github.com/larvit/fejkdata/internal/jsonvalue"
) )
// Template is an inline template compiled, referenced and validated against a // Template is an inline template compiled, referenced and validated against a
// generator's loaded data once, ready to render many times with [Template.Fake]. // generator's loaded data once, ready to render many times with [Template.Fake].
// It is safe for concurrent use: Fake serializes on its generator's lock, so a // It is safe for concurrent use: Fake serializes on its generator's lock, so a
// seeded sequence is reproducible only when a generator — and its templates — are // sequence is reproducible only when one goroutine makes every draw from a generator
// drawn from one goroutine. // and its templates.
type Template struct { type Template struct {
g *Generator g *Generator
n node n node
} }
// Fake renders the template with one draw. // Fake renders the template once; each call draws anew.
func (t *Template) Fake() string { func (t *Template) Fake() string {
t.g.mu.Lock() t.g.mu.Lock()
defer t.g.mu.Unlock() defer t.g.mu.Unlock()
return renderOnce(t.g.rand, t.n) return renderOnce(t.g.drawState, t.n)
} }
// NewTemplate compiles an inline template — a format string or a JSON value — and // NewTemplate compiles an inline template — a format string or a JSON value — and
// binds its references against the loaded tree, so repeated renders pay the // resolves its references against the loaded tree, so repeated renders pay the
// compile and validation once. It shares [New]'s guarantees: a bad template errors // compile and validation once. It shares [New]'s guarantees: a bad template errors
// here, and rendering cannot fail. // here, and rendering cannot fail.
func (f *Generator) NewTemplate(input string) (*Template, error) { func (f *Generator) NewTemplate(input string) (*Template, error) {
@@ -32,8 +34,10 @@ func (f *Generator) NewTemplate(input string) (*Template, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("fejkdata: %w", err) return nil, fmt.Errorf("fejkdata: %w", err)
} }
if err := bindInline(n, "template", f.categories, checkScope); err != nil { f.mu.Lock()
return nil, fmt.Errorf("fejkdata: %w", err) defer f.mu.Unlock()
if err := loadInline(&f.root, n, "template", false); err != nil {
return nil, f.noDataNote(fmt.Errorf("fejkdata: %w", err))
} }
return &Template{g: f, n: n}, nil return &Template{g: f, n: n}, nil
} }
@@ -49,127 +53,70 @@ func (f *Generator) FakeTemplate(input string) (string, error) {
return t.Fake(), nil return t.Fake(), nil
} }
// IsTemplate reports whether arg is an inline template rather than a path, by its shape: a { // IsTemplate reports whether arg is an inline template rather than a path, by its shape: an
// token, or a JSON object, array or string, is a template, and anything else is a path. A // arg holding a {, or a valid JSON array or string, is a template, and anything else is a
// name never holds a bracket, a brace or a quote, so an arg holding one that is not valid // path. A path errors where it holds a } or a quote, starts with [, or is no valid path. A path
// JSON names neither, and errors; so do a template of one reference alone, which is a path // may start with /, as a reference does, and every call taking a path drops it.
// written as a template, and a path written with a leading /.
func IsTemplate(arg string) (bool, error) { func IsTemplate(arg string) (bool, error) {
inline, err := isTemplate(arg) inline, err := grammar.IsTemplate(arg)
if err != nil { if err != nil {
return false, fmt.Errorf("fejkdata: %w", err) return false, fmt.Errorf("fejkdata: %w", err)
} }
return inline, nil return inline, nil
} }
func isTemplate(arg string) (bool, error) {
if strings.ContainsRune(arg, '{') || (isJSONStart(strings.TrimSpace(arg)) && json.Valid([]byte(arg))) {
if path, lone := loneReference(arg); lone {
return false, pathAdvice(path, fmt.Sprintf("%s is the path %s written as a template", arg, path))
}
return true, nil
}
if i := strings.IndexAny(arg, `}"`); i >= 0 {
return false, fmt.Errorf("%q holds a %q, which no path may, and it is not valid JSON, so it names no template either", arg, arg[i:i+1])
}
if strings.HasPrefix(strings.TrimSpace(arg), "[") {
return false, fmt.Errorf(`%q starts with "[" and is not valid JSON, so it names no template; a path starts with a name`, arg)
}
if _, err := splitPath(arg); err != nil {
return false, err
}
if path := strings.TrimLeft(arg, "/"); path != arg && path != "" {
return false, pathAdvice(path, fmt.Sprintf("path %s starts with /, and every path starts at the root already", arg))
}
return false, nil
}
func isJSONStart(arg string) bool {
return strings.HasPrefix(arg, "[") || strings.HasPrefix(arg, `"`)
}
// pathAdvice refuses a spelling of path, naming path to write, or why no name can spell it.
func pathAdvice(path, refusal string) error {
if err := checkPathNames(path); err != nil {
return err
}
return fmt.Errorf("%s; write %s", refusal, path)
}
// loneReference is the path a template spells when the value it holds — a format string, a
// JSON string, or an object holding only a format — is one reference token and nothing else.
func loneReference(arg string) (string, bool) {
var raw any
if json.Unmarshal([]byte(arg), &raw) != nil {
raw = arg
}
if m, isObject := raw.(map[string]any); isObject && len(m) == 1 {
raw = m["format"]
}
format, isString := raw.(string)
body, lone := loneRef(format)
if !isString || !lone {
return "", false
}
if strings.HasPrefix(body, "/") {
body = "/" + strings.TrimLeft(body, "/")
}
_, path, err := refShape(body)
return path, err == nil
}
func compileInput(input string) (node, error) { func compileInput(input string) (node, error) {
v, err := inputValue(input) return compile(inputValue(input))
}
// inputValue reads an inline template as the value compile takes: the JSON object, array, string
// or null it holds, the whitespace around it dropped, or else the input itself as a format string,
// a bare number or boolean among them.
func inputValue(input string) jsonvalue.Value {
raw, err := jsonvalue.Decode([]byte(input))
if err != nil { if err != nil {
return nil, err return jsonvalue.String(input)
} }
return compile(v) switch raw.Any().(type) {
case json.Number, bool:
return jsonvalue.String(input)
}
return raw
} }
// inputValue reads an inline template as the value compile takes: the JSON value it holds, or // loadInline loads the unloaded categories n reads, then runs n through the pipeline against root.
// the input itself as a format string when it is not JSON. func loadInline(root *folder, n node, label string, typedByGo bool) error {
func inputValue(input string) (any, error) { nodes := inlineNodes(n, label)
var raw any if err := refuseFolderRefs(nodes); err != nil {
if err := json.Unmarshal([]byte(input), &raw); err != nil {
return input, nil
}
if trimmed := strings.TrimSpace(input); trimmed != input {
return nil, fmt.Errorf("a JSON template may not be padded with spaces, which a format string would render; write %s", trimmed)
}
return raw, nil
}
// bindInline links an inline node's references against root and runs check over it, naming its
// nodes from label.
func bindInline(n node, label string, root map[string]node, check func(nodeScope) error) error {
if t, isTemplate := n.(*template); isTemplate && t.drawGroup != "" {
return fmt.Errorf("%s: drawGroup %q names nothing, since nothing can reference an inline template; drop it", label, t.drawGroup)
}
scope := inlineScope(n, label)
if err := linkNodeRefs(scope, root); err != nil {
return err return err
} }
return check(scope) if err := loadReached(root, nil, referenced(root, nil, nodes)); err != nil {
return loadError{err}
}
return pipeline{
nodes: nodes,
resolve: func() error { return resolveInlineTemplates(nodes, root.children) },
typedByGo: typedByGo,
}.run()
} }
// linkNodeRefs binds the references in an inline node's templates against the // refuseFolderRefs refuses each reference in nodes not written from the root, {/x}: an
// loaded tree. // inline node sits in no folder.
func linkNodeRefs(scope nodeScope, root map[string]node) error { func refuseFolderRefs(nodes nodeSet) error {
return scope(func(path string, m node) error { return nodes(func(label string, m node) error {
t, ok := m.(*template) t, ok := m.(*template)
if !ok { if !ok {
return nil return nil
} }
t.keyDrawGroup("") for _, name := range refTokens(t.tokens) {
for _, name := range refTokens(t.format) { sigil, rest, err := grammar.RefShape(name)
sigil, rest, err := refShape(name)
if err != nil { if err != nil {
return fmt.Errorf("%s: reference {%s}: %w", path, name, err) return fmt.Errorf("%s: reference {%s}: %w", label, name, err)
} }
if sigil != "/" { if sigil != "/" {
return fmt.Errorf("%s: reference {%s}: an inline template has no folder; write {/%s}", path, name, rest) return fmt.Errorf("%s: reference {%s}: an inline template has no folder; write {/%s}", label, name, rest)
} }
} }
return linkTemplateRefs(nil, path, "", t, root) return nil
}) })
} }
+44 -46
View File
@@ -16,7 +16,7 @@ func tmpl(t *testing.T, f *Generator, input string) string {
func shipped(t *testing.T, opts ...Option) *Generator { func shipped(t *testing.T, opts ...Option) *Generator {
t.Helper() t.Helper()
f, err := New(append([]Option{WithSeed(1)}, opts...)...) f, err := New(append([]Option{withShipped(), WithSeed(1)}, opts...)...)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -53,13 +53,13 @@ func TestFakeTemplateJSONArray(t *testing.T) {
} }
} }
func TestFakeTemplateCorrelatedReferences(t *testing.T) { func TestFakeTemplateNamedReference(t *testing.T) {
dir := writeData(t, map[string]string{ dir := writeData(t, map[string]string{
"person": `[{"format":"{first} {last}","first":"Ada","last":"Lovelace"},{"format":"{first} {last}","first":"Bo","last":"Ek","born":"1990"}]`, "person": `[{"format":"{first} {last}","first":"Ada","last":"Lovelace"},{"format":"{first} {last}","first":"Bo","last":"Ek","born":"1990"}]`,
}) })
f := newGenerator(t, dir, WithSeed(1)) f := newGenerator(t, dir, WithSeed(1))
for i := 0; i < 100; i++ { for i := 0; i < 100; i++ {
got := tmpl(t, f, "{/person.first} {/person.last}") got := tmpl(t, f, "{/person as p}{p.first} {p.last}")
if got != "Ada Lovelace" && got != "Bo Ek" { if got != "Ada Lovelace" && got != "Bo Ek" {
t.Fatalf("correlated references = %q, want one person's first and last", got) t.Fatalf("correlated references = %q, want one person's first and last", got)
} }
@@ -104,13 +104,9 @@ func TestFakeTemplateErrors(t *testing.T) {
}{ }{
{`{"x":"Q"}`, "missing string \"format\""}, {`{"x":"Q"}`, "missing string \"format\""},
{`"{x}"`, `no field "x"`}, {`"{x}"`, `no field "x"`},
{`"{digits(0)}"`, "must be positive"}, {`"{digits(-1)}"`, "must not be negative"},
{`name: {/no.such.path}`, "no entry"}, {`name: {/no.such.path}`, "no entry"},
{`name: {..nope}`, "write {/nope}"}, {`name: {..nope}`, "write {/nope}"},
{`{"format":"x"}`, "is a string"},
{`{/misc.territory} {/misc.territory.alpha2}`, "renders a level that {/misc.territory.alpha2} reads a path into; name the fields you want instead, or draw them apart with a drawGroup"},
{`{"format":"{/misc.territory.alpha2} {x}","x":"{/misc.territory}"}`, "reads a path into"},
{`{"format":"{/misc.territory.alpha2}","drawGroup":"g"}`, "nothing can reference"},
} { } {
_, err := f.FakeTemplate(c.input) _, err := f.FakeTemplate(c.input)
if err == nil || !strings.Contains(err.Error(), c.want) { if err == nil || !strings.Contains(err.Error(), c.want) {
@@ -127,12 +123,15 @@ func TestFakeTemplateJSONString(t *testing.T) {
} }
} }
func TestPaddedJSONIsRejected(t *testing.T) { func TestPaddedJSONIsTheJSON(t *testing.T) {
f := shipped(t) f := shipped(t)
in := `{"format":"{x}","x":["a","b"]}` in := `{"format":"{x}","x":["a","b"]}`
_, err := f.NewTemplate(" " + in + " ") padded, err := f.NewTemplate(" \t" + in + "\n ")
if err == nil || !strings.Contains(err.Error(), "write "+in) { if err != nil {
t.Fatalf("NewTemplate(padded JSON) = %v, want an error naming the unpadded spelling", err) t.Fatalf("NewTemplate(padded JSON) = %v, want the JSON it pads", err)
}
if got := padded.Fake(); got != "a" && got != "b" {
t.Fatalf("padded JSON rendered %q, want a or b with no padding", got)
} }
} }
@@ -153,46 +152,45 @@ func TestNewTemplateReusable(t *testing.T) {
func TestIsTemplate(t *testing.T) { func TestIsTemplate(t *testing.T) {
for arg, want := range map[string]bool{ for arg, want := range map[string]bool{
"sv_SE.person": false, "sv_SE.person": false,
"person.last": false, "person.last": false,
"name: {x}": true, "name: {x}": true,
`{"format":"x"}`: true, `{"format":"x"}`: true,
`["a","b"]`: true, `["a","b"]`: true,
`[1, 2]`: true, `[1, 2]`: true,
` ["a","b"]`: true, // padding is the template's own error, not a shape verdict ` ["a","b"]`: true, // padding is the template's own error, not a shape verdict
`"hello"`: true, `"hello"`: true,
"{/a}{/b}": true, "{/a}{/b}": true,
"{/a|/b}": true, "{/a|/b}": true,
"{uppercase(/a)}": true, "{uppercase(/a)}": true,
"{{/a}}": true, "{{/a}}": true,
`"{/a} x"`: true, `"{/a} x"`: true,
"x[1]": false, // a row selected by key or name "x[1]": false, // a row selected by key or name
"x[St. Louis].y": false, "x[St. Louis].y": false,
"{/sv_SE.person.last}": true,
`"{/sv_SE.person}"`: true,
"{.person.last}": true,
` "{/sv_SE.person}"`: true,
`{"format":"{/sv_SE.person}"}`: true,
"{//sv_SE.person}": true,
"{/misc.currency[US Dollar (Next day)]}": true,
"/sv_SE.person": false,
} { } {
if got, err := IsTemplate(arg); err != nil || got != want { if got, err := IsTemplate(arg); err != nil || got != want {
t.Errorf("IsTemplate(%q) = %v, %v; want %v", arg, got, err, want) t.Errorf("IsTemplate(%q) = %v, %v; want %v", arg, got, err, want)
} }
} }
for arg, want := range map[string]string{ for arg, want := range map[string]string{
"[abc]": `starts with "["`, "[abc]": `starts with "["`,
"[abc].field": `starts with "["`, "[abc].field": `starts with "["`,
"x[1]y": `"]"`, "x[1]y": `"]"`,
"x[[1]]": `"["`, "x[[1]]": `"["`,
"x[]": "empty", "x[]": "empty",
"a]b": `holds a "]"`, "//sv_SE.person": "write /sv_SE.person",
"a}b": `holds a "}"`, "a]b": `holds a "]"`,
`"abc`: `holds a "\""`, "a}b": `holds a "}"`,
`"a]b`: `holds a "\""`, // the opener the reader typed, not the bracket behind it `"abc`: `holds a "\""`,
"{/sv_SE.person.last}": "{/sv_SE.person.last} is the path sv_SE.person.last written as a template; write sv_SE.person.last", `"a]b`: `holds a "\""`, // the opener the reader typed, not the bracket behind it
`"{/sv_SE.person}"`: "write sv_SE.person",
"{.person.last}": "write person.last",
"/sv_SE.person": "write sv_SE.person",
` "{/sv_SE.person}"`: "write sv_SE.person",
`{"format":"{/sv_SE.person}"}`: "write sv_SE.person",
"//sv_SE.person": "write sv_SE.person",
"{//sv_SE.person}": "{//sv_SE.person} is the path sv_SE.person written as a template; write sv_SE.person",
"/sv_SE/person": `path "sv_SE/person" contains "/"`,
"{/-}": `path "-" is reserved`,
} { } {
if _, err := IsTemplate(arg); err == nil || !strings.Contains(err.Error(), want) { if _, err := IsTemplate(arg); err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("IsTemplate(%q) = %v; want it rejected naming %s", arg, err, want) t.Errorf("IsTemplate(%q) = %v; want it rejected naming %s", arg, err, want)
+119
View File
@@ -0,0 +1,119 @@
// Package builtinfunc holds the {name(args)} functions a format calls: their checks, the
// draws they make, and the numbers their text is proven to read as.
package builtinfunc
import (
"fmt"
"slices"
"strings"
"github.com/larvit/fejkdata/internal/datatype"
"github.com/larvit/fejkdata/internal/drawstate"
"github.com/larvit/fejkdata/internal/proven"
)
// Call is a builtin prepared for one call site: its args already parsed. It reads the
// output emitted so far in the current expansion (a derivation's payload) and the
// values of the operands it named, which the caller read for it.
type Call func(s *drawstate.State, emitted string, operands []string) string
// builtin is a format-string function invoked as {name(args)}. Its Call stays pure over its
// inputs, so seeded output is reproducible. arity is the exact arg count, or -1 for variadic,
// where checkArgs does all the validation.
type builtin struct {
arity int
// prep parses validated args once, at compile time, into the closure a render calls.
prep func(args []string) Call
checkArgs func(args []string) error
// operands names the fields the call reads, which a render reads for it; nil
// for a builtin that reads none.
operands func(args []string) []string
// proveNumber bounds the number a call's text reads as, token its body, and says which
// datatypes that text is not; set it where every render reads as a finite number, which
// makes the call a calc operand, and leave it nil otherwise.
proveNumber func(token string, prints datatype.DataType, args []string) proven.Facts
// prints is the datatype a call's text is, handed to proveNumber: datatype.String where it
// reads as a number no column should type, as digits' leading zeros; unset where
// proveNumber is nil.
prints datatype.DataType
}
// CalcName is the builtin whose operands package fejkdata checks and proves, since only it
// holds the nodes they read.
const CalcName = "calc"
// rng is the randomness a builtin sample draws from, which *drawstate.State satisfies.
type rng interface {
IntN(n int) int
Float64() float64
}
// Check validates a call at compile time: naming a known builtin, with the arg count that
// builtin takes and args its check accepts.
func Check(name string, args []string) error {
b, known := builtins[name]
if !known {
return fmt.Errorf("unknown function %q", name)
}
if b.arity >= 0 && len(args) != b.arity {
return fmt.Errorf("%s takes %d argument%s, got %d", name, b.arity, plural(b.arity), len(args))
}
if b.checkArgs != nil {
return b.checkArgs(args)
}
return nil
}
// Prep parses a call's args, which Check accepted, into the Call a render makes.
func Prep(name string, args []string) Call { return builtins[name].prep(args) }
// Operands lists the fields a call reads as operands, empty for a builtin that reads none
// or is unknown.
func Operands(name string, args []string) []string {
b, known := builtins[name]
if !known || b.operands == nil {
return nil
}
return b.operands(args)
}
// IsTransform reports a builtin that rewrites the text of its one operand.
func IsTransform(name string) bool {
_, isTransform := transforms[name]
return isTransform
}
// ProveNumber proves a call whose every render reads as a finite number, token its body;
// false for any other call, calc included: proven.OfCalc proves a calc from its operands,
// which only the caller holds.
func ProveNumber(name, token string, args []string) (proven.Facts, bool) {
b := builtins[name]
if b.proveNumber == nil {
return proven.Facts{}, false
}
return b.proveNumber(token, b.prints, args), true
}
// TypedCalls lists the calls whose text a typed column may hold, as an error names them.
func TypedCalls() string { return numberCalls(false) }
// OperandCalls lists the calls whose text a calc reads as a number, as an error names them.
func OperandCalls() string { return numberCalls(true) }
func numberCalls(text bool) string {
var calls []string
for name, b := range builtins {
if b.proveNumber != nil && (text || b.prints != datatype.String) {
calls = append(calls, "{"+name+"()}")
}
}
slices.Sort(calls)
return strings.Join(calls, ", ") + " or {" + CalcName + "()}"
}
func plural(n int) string {
if n == 1 {
return ""
}
return "s"
}
+68
View File
@@ -0,0 +1,68 @@
package builtinfunc
import (
"fmt"
"strings"
"testing"
"github.com/larvit/fejkdata/internal/invariant"
)
// TestRegistryShapes pins the builtin contract Prep relies on: every entry supplies prep,
// and args parsed at compile only behind a check.
func TestRegistryShapes(t *testing.T) {
for name, b := range builtins {
if b.prep == nil {
t.Errorf("builtin %q has no prep: Prep would call a nil func", name)
}
if b.arity != 0 && b.checkArgs == nil {
t.Errorf("builtin %q parses args in prep with no check", name)
}
}
}
func TestCheckNamesWhatACallGetsWrong(t *testing.T) {
for _, c := range []struct {
name string
args []string
want string
}{
{"nope", nil, `unknown function "nope"`},
{"luhn", []string{"1"}, "luhn takes 0 arguments, got 1"},
{"hex", nil, "hex takes 1 argument, got 0"},
{"calc", []string{"a +"}, `calc("a +"): `},
{"calc", []string{"a", "-1"}, "calc decimals "},
{"calc", nil, "calc takes an expression and an optional decimals count, got 0 args"},
} {
if err := Check(c.name, c.args); err == nil || !strings.HasPrefix(err.Error(), c.want) {
t.Errorf("Check(%s, %q) = %v, want it to start %q", c.name, c.args, err, c.want)
}
}
if err := Check("calc", []string{"a / b", "2"}); err != nil {
t.Errorf("Check(calc) = %v, want an expression over any operand accepted", err)
}
}
// TestArgGuardsPanic pins the guards that report a builtin arg its check should have
// rejected. No data reaches them, since Check runs before Prep, so they are called directly.
func TestArgGuardsPanic(t *testing.T) {
for name, call := range map[string]func(){
"atoi on an unvalidated arg": func() { atoi("nope") },
"atof on an unvalidated arg": func() { atof("nope") },
"prep on an expression that won't parse": func() { calcPrep([]string{"1 +"}) },
} {
mustPanic(t, name, call)
}
}
// mustPanic fails unless call panics with invariant.Broken's phrase, which is what
// separates a reported invariant break from a silently wrong value.
func mustPanic(t *testing.T, name string, call func()) {
t.Helper()
defer func() {
if r := recover(); r == nil || !strings.HasPrefix(fmt.Sprint(r), invariant.Broken("")) {
t.Errorf("%s: recovered %v, want the invariant reported as an internal error", name, r)
}
}()
call()
}
@@ -1,4 +1,4 @@
package fejkdata package builtinfunc
import ( import (
"encoding/base64" "encoding/base64"
@@ -7,6 +7,11 @@ import (
"math" "math"
"strconv" "strconv"
"strings" "strings"
"github.com/larvit/fejkdata/internal/datatype"
"github.com/larvit/fejkdata/internal/drawstate"
"github.com/larvit/fejkdata/internal/invariant"
"github.com/larvit/fejkdata/internal/proven"
) )
// maxLen caps sample output lengths (hex, nanoid, base64, digits, upper, lower) // maxLen caps sample output lengths (hex, nanoid, base64, digits, upper, lower)
@@ -22,88 +27,88 @@ const (
// emitted so far in the current expansion (place them after their payload); // emitted so far in the current expansion (place them after their payload);
// samples read only the rng. A time-based id (uuid v7, ulid) draws its timestamp // samples read only the rng. A time-based id (uuid v7, ulid) draws its timestamp
// from the rng, not the wall clock, so seeded output stays reproducible. // from the rng, not the wall clock, so seeded output stays reproducible.
var builtins = map[string]builtin{ var builtins = withTransforms(map[string]builtin{
"luhn": {arity: 0, prep: derive(func(e string) string { return string(rune('0' + luhnCheck(e))) })}, "luhn": {arity: 0, prep: derive(func(e string) string { return string(rune('0' + luhnCheck(e))) })},
"mod11": {arity: 0, prep: derive(mod11Check)}, "mod11": {arity: 0, prep: derive(mod11Check)},
"ean": {arity: 0, prep: derive(eanCheck)}, "ean": {arity: 0, prep: derive(eanCheck)},
"uuid": {arity: 0, prep: sample(uuidV7)}, "uuid": {arity: 0, prep: sample(uuidV7)},
"ulid": {arity: 0, prep: sample(ulid)}, "ulid": {arity: 0, prep: sample(ulid)},
"nanoid": {arity: 1, check: posIntArg, prep: chars(nanoidAlphabet)}, "nanoid": {arity: 1, checkArgs: countArg, prep: chars(nanoidAlphabet)},
"hex": {arity: 1, check: posIntArg, prep: chars(hexDigits)}, "hex": {arity: 1, checkArgs: countArg, prep: chars(hexDigits)},
"digits": {arity: 1, check: posIntArg, prep: chars("0123456789"), number: func(token string, a []string) proven { "digits": {arity: 1, checkArgs: countArg, prep: chars("0123456789"), prints: datatype.String, proveNumber: func(token string, prints datatype.DataType, a []string) proven.Facts {
return printing(token, DataTypeString, bounded(0, math.Pow(10, float64(atoi(a[0])))-1, true)) if atoi(a[0]) == 0 {
return proven.Unproven("{" + token + "} prints nothing, which is no number")
}
return proven.Printing(token, prints, proven.Bounded(0, math.Pow(10, float64(atoi(a[0])))-1, true))
}}, }},
"upper": {arity: 1, check: posIntArg, prep: chars("ABCDEFGHIJKLMNOPQRSTUVWXYZ")}, "upper": {arity: 1, checkArgs: countArg, prep: chars("ABCDEFGHIJKLMNOPQRSTUVWXYZ")},
"lower": {arity: 1, check: posIntArg, prep: chars("abcdefghijklmnopqrstuvwxyz")}, "lower": {arity: 1, checkArgs: countArg, prep: chars("abcdefghijklmnopqrstuvwxyz")},
"base64": {arity: 1, check: posIntArg, prep: func(a []string) callFn { "base64": {arity: 1, checkArgs: countArg, prep: func(a []string) Call {
n := atoi(a[0]) n := atoi(a[0])
return func(s *session, _ string, _ []string) string { return func(s *drawstate.State, _ string, _ []string) string {
return base64.StdEncoding.EncodeToString(randBytes(s, n)) return base64.StdEncoding.EncodeToString(randBytes(s, n))
} }
}}, }},
"int": {arity: 2, check: intRangeArgs, prep: func(a []string) callFn { "int": {arity: 2, checkArgs: intRangeArgs, prep: func(a []string) Call {
lo, span := atoi(a[0]), atoi(a[1])-atoi(a[0])+1 lo, span := atoi(a[0]), atoi(a[1])-atoi(a[0])+1
return func(s *session, _ string, _ []string) string { return strconv.Itoa(lo + s.IntN(span)) } return func(s *drawstate.State, _ string, _ []string) string { return strconv.Itoa(lo + s.IntN(span)) }
}, number: func(token string, a []string) proven { }, prints: datatype.Integer, proveNumber: func(token string, prints datatype.DataType, a []string) proven.Facts {
return printing(token, DataTypeInteger, bounded(float64(atoi(a[0])), float64(atoi(a[1])), true)) return proven.Printing(token, prints, proven.Bounded(float64(atoi(a[0])), float64(atoi(a[1])), true))
}}, }},
"float": {arity: 3, check: floatArgs, prep: func(a []string) callFn { "float": {arity: 3, checkArgs: floatArgs, prep: func(a []string) Call {
lo, hi, dp := atof(a[0]), atof(a[1]), atoi(a[2]) lo, hi, dp := atof(a[0]), atof(a[1]), atoi(a[2])
return func(s *session, _ string, _ []string) string { return func(s *drawstate.State, _ string, _ []string) string {
return formatFloat(lo+s.Float64()*(hi-lo), dp) return formatFloat(lo+s.Float64()*(hi-lo), dp)
} }
}, number: func(token string, a []string) proven { }, prints: datatype.Number, proveNumber: func(token string, _ datatype.DataType, a []string) proven.Facts {
return printedNumber(token, bounded(atof(a[0]), atof(a[1]), false), atoi(a[2])) return proven.PrintedNumber(token, proven.Bounded(atof(a[0]), atof(a[1]), false), atoi(a[2]))
}}, }},
"iban": {arity: 1, check: ibanArg, prep: func(a []string) callFn { "iban": {arity: 1, checkArgs: ibanArg, prep: func(a []string) Call {
cc := a[0] cc := a[0]
return func(s *session, _ string, _ []string) string { return iban(s, cc) } return func(s *drawstate.State, _ string, _ []string) string { return iban(s, cc) }
}}, }},
"date": {arity: -1, check: dateArgs, prep: datePrep}, "date": {arity: -1, checkArgs: dateArgs, prep: datePrep},
"time": {arity: -1, check: timeArg, prep: timePrep}, "time": {arity: -1, checkArgs: timeArg, prep: timePrep},
"calc": {arity: -1, check: checkCalc, prep: calcPrep, operands: calcOperands}, CalcName: {arity: -1, checkArgs: checkCalc, prep: calcPrep, operands: calcOperands},
"lowercase": {arity: 1, check: transformArg, prep: transformPrep(strings.ToLower), operands: transformOperand}, // seq is the one stateful builtin: a per-generator counter from 1, advancing on
"uppercase": {arity: 1, check: transformArg, prep: transformPrep(strings.ToUpper), operands: transformOperand},
"ascii": {arity: 1, check: transformArg, prep: transformPrep(asciiFold), operands: transformOperand},
// seq is the one stateful builtin: a per-session counter from 1, advancing on
// each call. An optional name selects an independent counter; no name uses the // each call. An optional name selects an independent counter; no name uses the
// default one. Deterministic by construction, so seeded output stays stable. // default one. Deterministic by construction, so seeded output stays stable.
"seq": {arity: -1, check: seqArg, prep: func(a []string) callFn { "seq": {arity: -1, checkArgs: seqArg, prep: func(a []string) Call {
key := "" key := ""
if len(a) == 1 { if len(a) == 1 {
key = a[0] key = a[0]
} }
return func(s *session, _ string, _ []string) string { return func(s *drawstate.State, _ string, _ []string) string {
return strconv.FormatUint(s.next(key), 10) return strconv.FormatUint(s.Seq(key), 10)
} }
}, number: func(token string, _ []string) proven { }, prints: datatype.Integer, proveNumber: func(token string, prints datatype.DataType, _ []string) proven.Facts {
return printing(token, DataTypeInteger, bounded(1, math.MaxInt64, true)) return proven.Printing(token, prints, proven.Bounded(1, math.MaxInt64, true))
}}, }},
} })
// derive and sample are the two argument-free builtin shapes: a derivation reads // derive and sample are the two argument-free builtin shapes: a derivation reads
// the output emitted so far, a sample reads only the rng. chars is the shape of a // the output emitted so far, a sample reads only the rng. chars is the shape of a
// sample of n characters drawn from an alphabet. // sample of n characters drawn from an alphabet.
func derive(f func(emitted string) string) func([]string) callFn { func derive(f func(emitted string) string) func([]string) Call {
return func([]string) callFn { return func([]string) Call {
return func(_ *session, emitted string, _ []string) string { return f(emitted) } return func(_ *drawstate.State, emitted string, _ []string) string { return f(emitted) }
} }
} }
func sample(f func(rng) string) func([]string) callFn { func sample(f func(rng) string) func([]string) Call {
return func([]string) callFn { return func([]string) Call {
return func(s *session, _ string, _ []string) string { return f(s) } return func(s *drawstate.State, _ string, _ []string) string { return f(s) }
} }
} }
func chars(alphabet string) func([]string) callFn { func chars(alphabet string) func([]string) Call {
return func(a []string) callFn { return func(a []string) Call {
n := atoi(a[0]) n := atoi(a[0])
return func(s *session, _ string, _ []string) string { return randChars(s, n, alphabet) } return func(s *drawstate.State, _ string, _ []string) string { return randChars(s, n, alphabet) }
} }
} }
const hexDigits = "0123456789abcdef" const hexDigits = "0123456789abcdef"
// formatFloat prints v to dp decimals, -1 for the shortest form, and a zero unsigned. // formatFloat prints v to dp decimals, or proven.ShortestDecimals, and a zero unsigned.
func formatFloat(v float64, dp int) string { func formatFloat(v float64, dp int) string {
s := strconv.FormatFloat(v, 'f', dp, 64) s := strconv.FormatFloat(v, 'f', dp, 64)
if strings.HasPrefix(s, "-") && strings.Trim(s, "-0.") == "" { if strings.HasPrefix(s, "-") && strings.Trim(s, "-0.") == "" {
@@ -118,7 +123,7 @@ func formatFloat(v float64, dp int) string {
func atoi(s string) int { func atoi(s string) int {
n, err := strconv.Atoi(s) n, err := strconv.Atoi(s)
if err != nil { if err != nil {
panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", s, err)) panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", s, err))
} }
return n return n
} }
@@ -127,7 +132,7 @@ func atoi(s string) int {
func atof(s string) float64 { func atof(s string) float64 {
f, err := strconv.ParseFloat(s, 64) f, err := strconv.ParseFloat(s, 64)
if err != nil { if err != nil {
panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", s, err)) panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", s, err))
} }
return f return f
} }
@@ -146,8 +151,7 @@ func randChars(r rng, n int, alphabet string) string {
return string(b) return string(b)
} }
// plainInt parses an integer arg written the one way: no sign, no leading zero. func intArg(s string) (int, error) {
func plainInt(s string) (int, error) {
n, err := strconv.Atoi(s) n, err := strconv.Atoi(s)
if errors.Is(err, strconv.ErrRange) { if errors.Is(err, strconv.ErrRange) {
return 0, fmt.Errorf("%q is past the integer range: %w", s, err) return 0, fmt.Errorf("%q is past the integer range: %w", s, err)
@@ -155,54 +159,48 @@ func plainInt(s string) (int, error) {
if err != nil { if err != nil {
return 0, fmt.Errorf("%q is not an integer", s) return 0, fmt.Errorf("%q is not an integer", s)
} }
if strconv.Itoa(n) != s {
return 0, fmt.Errorf("%q is not a plain integer; write %d", s, n)
}
return n, nil return n, nil
} }
func posIntArg(_ map[string]node, a []string) error { func countArg(a []string) error {
n, err := plainInt(a[0]) n, err := intArg(a[0])
if errors.Is(err, strconv.ErrRange) { if errors.Is(err, strconv.ErrRange) {
return fmt.Errorf("count %q exceeds the maximum %d", a[0], maxLen) return fmt.Errorf("count %q exceeds the maximum %d", a[0], maxLen)
} }
if err != nil { if err != nil {
return fmt.Errorf("count %w", err) return fmt.Errorf("count %w", err)
} }
if n < 1 { if n < 0 {
return fmt.Errorf("count %q must be positive", a[0]) return fmt.Errorf("count %q must not be negative", a[0])
} }
if n > maxLen { if n > maxLen {
return fmt.Errorf("count %d exceeds the maximum %d", n, maxLen) return fmt.Errorf("count %d exceeds the maximum %d", n, maxLen)
} }
return nil return nil
} }
func intRangeArgs(_ map[string]node, a []string) error { func intRangeArgs(a []string) error {
lo, err := plainInt(a[0]) lo, err := intArg(a[0])
if err != nil { if err != nil {
return fmt.Errorf("int(min,max): min %w", err) return fmt.Errorf("int(min,max): min %w", err)
} }
hi, err := plainInt(a[1]) hi, err := intArg(a[1])
if err != nil { if err != nil {
return fmt.Errorf("int(min,max): max %w", err) return fmt.Errorf("int(min,max): max %w", err)
} }
if lo > hi { if lo > hi {
return fmt.Errorf("int(min,max): min %d > max %d", lo, hi) return fmt.Errorf("int(min,max): min %d > max %d", lo, hi)
} }
if lo == hi {
return fmt.Errorf("int(%d,%d) is the constant %d; write it as text", lo, hi, lo)
}
if uint64(hi)-uint64(lo) >= uint64(math.MaxInt64) { // span hi-lo+1 would overflow int -> IntN panic if uint64(hi)-uint64(lo) >= uint64(math.MaxInt64) { // span hi-lo+1 would overflow int -> IntN panic
return fmt.Errorf("int(min,max): range %d..%d is too wide", lo, hi) return fmt.Errorf("int(min,max): range %d..%d is too wide", lo, hi)
} }
return nil return nil
} }
func floatArgs(_ map[string]node, a []string) error { func floatArgs(a []string) error {
lo, e1 := strconv.ParseFloat(a[0], 64) lo, e1 := strconv.ParseFloat(a[0], 64)
hi, e2 := strconv.ParseFloat(a[1], 64) hi, e2 := strconv.ParseFloat(a[1], 64)
if e1 != nil || e2 != nil { if e1 != nil || e2 != nil {
return fmt.Errorf("float(min,max,dp) needs numeric bounds, got %q,%q", a[0], a[1]) return fmt.Errorf("float(min,max,dp) needs numeric bounds, got %q,%q", a[0], a[1])
} }
dp, err := plainInt(a[2]) dp, err := intArg(a[2])
if err != nil { if err != nil {
return fmt.Errorf("float(min,max,dp): decimals %w", err) return fmt.Errorf("float(min,max,dp): decimals %w", err)
} }
@@ -215,15 +213,12 @@ func floatArgs(_ map[string]node, a []string) error {
if dp < 0 || dp > maxDecimals { if dp < 0 || dp > maxDecimals {
return fmt.Errorf("float(min,max,dp): decimals %d out of range 0..%d", dp, maxDecimals) return fmt.Errorf("float(min,max,dp): decimals %d out of range 0..%d", dp, maxDecimals)
} }
if lo == hi {
return fmt.Errorf("float(%s,%s,%d) is the constant %q; write it as text", a[0], a[1], dp, strconv.FormatFloat(lo, 'f', dp, 64))
}
if math.IsInf(hi-lo, 0) { // an overflowing span would render as "+Inf" if math.IsInf(hi-lo, 0) { // an overflowing span would render as "+Inf"
return fmt.Errorf("float(min,max,dp): range %v..%v is too wide", lo, hi) return fmt.Errorf("float(min,max,dp): range %v..%v is too wide", lo, hi)
} }
return nil return nil
} }
func seqArg(_ map[string]node, a []string) error { func seqArg(a []string) error {
if len(a) > 1 { if len(a) > 1 {
return fmt.Errorf("seq takes at most one name, got %d", len(a)) return fmt.Errorf("seq takes at most one name, got %d", len(a))
} }
+109
View File
@@ -0,0 +1,109 @@
package builtinfunc
import (
"fmt"
"math"
"strconv"
"strings"
"github.com/larvit/fejkdata/internal/drawstate"
"github.com/larvit/fejkdata/internal/grammar"
"github.com/larvit/fejkdata/internal/invariant"
"github.com/larvit/fejkdata/internal/proven"
)
// evalCalc evaluates an expression over the operand values a render already read, so it
// draws nothing.
func evalCalc(n grammar.CalcNode, operands []string) float64 {
switch n := n.(type) {
case grammar.CalcNum:
return float64(n)
case grammar.CalcVar:
v, err := strconv.ParseFloat(strings.TrimSpace(operands[n.At]), 64)
if err != nil {
return math.NaN() // a non-numeric operand stays visible, never an error
}
return v
case grammar.CalcNeg:
return -evalCalc(n.X, operands)
case grammar.CalcBin:
return Arith(n.Operator, evalCalc(n.L, operands), evalCalc(n.R, operands))
}
panic(invariant.Broken("calc node %#v has no evaluation", n))
}
// Arith applies a calc operator.
func Arith(operator byte, l, r float64) float64 {
switch operator {
case '+':
return l + r
case '-':
return l - r
case '*':
return l * r
default: // '/'
return l / r
}
}
// checkCalc validates a calc token's own args: a parseable expression and an optional
// non-negative integer dp. What its operands hold is the caller's to check.
func checkCalc(args []string) error {
if len(args) < 1 || len(args) > 2 {
return fmt.Errorf("calc takes an expression and an optional decimals count, got %d args", len(args))
}
if _, err := grammar.ParseCalc(args[0]); err != nil {
return fmt.Errorf("calc(%q): %w", args[0], err)
}
if len(args) == 2 {
dp, err := intArg(args[1])
if err != nil {
return fmt.Errorf("calc decimals %w", err)
}
if dp < 0 || dp > maxDecimals {
return fmt.Errorf("calc decimals %d must be in 0..%d", dp, maxDecimals)
}
}
return nil
}
// calcPrep parses the expression and decimals once, at compile time. checkCalc proved
// both args valid, so no step here can fail.
func calcPrep(args []string) Call {
expr := ParsedCalc(args[0]).Expr
dp := CalcDecimals(args)
return func(_ *drawstate.State, _ string, operands []string) string {
return formatFloat(evalCalc(expr, operands), dp)
}
}
// ParsedCalc parses an expression Check accepted.
func ParsedCalc(expr string) grammar.Calc {
c, err := grammar.ParseCalc(expr)
if err != nil {
panic(invariant.Broken("calc(%q) passed its check unparsed: %v", expr, err))
}
return c
}
// CalcDecimals is the decimals count of a calc Check accepted, or proven.ShortestDecimals
// where it names none.
func CalcDecimals(args []string) int {
if len(args) == 2 {
return atoi(args[1])
}
return proven.ShortestDecimals
}
// calcOperands lists the operands a calc's args read, fields or names. checkCalc reports
// an expression that does not parse, so one that does not simply names nothing.
func calcOperands(args []string) []string {
if len(args) == 0 {
return nil
}
c, err := grammar.ParseCalc(args[0])
if err != nil {
return nil
}
return c.Operands
}
@@ -1,4 +1,4 @@
package fejkdata package builtinfunc
import "fmt" import "fmt"
@@ -65,16 +65,14 @@ func eanCheck(s string) string {
// a left-to-right derivation; it generates the whole value instead. // a left-to-right derivation; it generates the whole value instead.
var ibanLen = map[string]int{"BE": 16, "DE": 22, "DK": 18, "ES": 24, "FI": 18, "NO": 15, "SE": 24} var ibanLen = map[string]int{"BE": 16, "DE": 22, "DK": 18, "ES": 24, "FI": 18, "NO": 15, "SE": 24}
func ibanArg(_ map[string]node, a []string) error { func ibanArg(a []string) error {
if _, ok := ibanLen[a[0]]; !ok { if _, ok := ibanLen[a[0]]; !ok {
return fmt.Errorf("iban(%q): unsupported country code", a[0]) return fmt.Errorf("iban(%q): unsupported country code", a[0])
} }
return nil return nil
} }
// iban generates a structurally valid IBAN for cc: a numeric BBAN of the right // iban generates an IBAN for cc: a numeric BBAN of the right length, then mod-97 check digits.
// length, then mod-97 check digits. Real bank/branch structure isn't modelled —
// the result passes length and checksum validation, which is what fake data needs.
func iban(r rng, cc string) string { func iban(r rng, cc string) string {
bban := make([]byte, ibanLen[cc]-4) bban := make([]byte, ibanLen[cc]-4)
for i := range bban { for i := range bban {
+27 -53
View File
@@ -1,39 +1,23 @@
package fejkdata package builtinfunc
import ( import (
"fmt" "fmt"
"strings" "strings"
"time" "time"
"github.com/larvit/fejkdata/internal/drawstate"
"github.com/larvit/fejkdata/internal/invariant"
) )
const dayLayout = "2006-01-02" const dayLayout = "2006-01-02"
// The instants a layout is proved against: layoutProbe2 is alike in no field, while
// layoutDay differs from layoutProbe in its date fields alone and layoutClock in its
// clock fields alone, so formatting two of them tells which kind a layout names.
var (
layoutProbe = time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
layoutProbe2 = time.Date(2010, 11, 12, 13, 14, 15, 0, time.UTC)
layoutDay = time.Date(2010, 11, 12, 4, 5, 6, 0, time.UTC)
layoutClock = time.Date(2001, 2, 3, 13, 14, 15, 0, time.UTC)
)
func namesAField(layout string) bool {
return layoutProbe.Format(layout) != layoutProbe2.Format(layout)
}
func namesADateField(layout string) bool {
return layoutProbe.Format(layout) != layoutDay.Format(layout)
}
func namesAClockField(layout string) bool {
return layoutProbe.Format(layout) != layoutClock.Format(layout)
}
// quotedLayout reports whether an arg carries the single quotes a layout is written in. // quotedLayout reports whether an arg carries the single quotes a layout is written in.
func quotedLayout(a string) bool { func quotedLayout(a string) bool {
return len(a) >= 2 && a[0] == '\'' && a[len(a)-1] == '\'' return len(a) >= 2 && a[0] == '\'' && a[len(a)-1] == '\''
} }
// layoutArg is the Go layout a quoted arg holds, refused when unquoted or constant. // layoutArg is the Go layout a quoted arg holds, refused when unquoted.
// docs/decisions.md#a-layout-is-always-quoted
func layoutArg(a string) (string, error) { func layoutArg(a string) (string, error) {
if !quotedLayout(a) { if !quotedLayout(a) {
bare := strings.Trim(a, `'"`) bare := strings.Trim(a, `'"`)
@@ -42,18 +26,14 @@ func layoutArg(a string) (string, error) {
} }
return "", fmt.Errorf("layout %s is not quoted; write '%s'", a, bare) return "", fmt.Errorf("layout %s is not quoted; write '%s'", a, bare)
} }
layout := a[1 : len(a)-1] return a[1 : len(a)-1], nil
if !namesAField(layout) {
return "", fmt.Errorf("layout '%s' names no field, so it is the constant %q; write it as text", layout, layout)
}
return layout, nil
} }
// layoutOf is layoutArg for an arg a check already validated. // layoutOf is layoutArg for an arg a check already validated.
func layoutOf(a string) string { func layoutOf(a string) string {
layout, err := layoutArg(a) layout, err := layoutArg(a)
if err != nil { if err != nil {
panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", a, err)) panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", a, err))
} }
return layout return layout
} }
@@ -81,7 +61,7 @@ func holdsQuotedLayout(a []string) bool {
} }
return false return false
} }
func dateArgs(_ map[string]node, a []string) error { func dateArgs(a []string) error {
if err := layoutArity("date", 3, a); err != nil { if err := layoutArity("date", 3, a); err != nil {
return err return err
} }
@@ -96,51 +76,45 @@ func dateArgs(_ map[string]node, a []string) error {
if to.Before(from) { if to.Before(from) {
return fmt.Errorf("date(from,to,layout): from %s is after to %s", a[0], a[1]) return fmt.Errorf("date(from,to,layout): from %s is after to %s", a[0], a[1])
} }
layout, err := layoutArg(a[2]) _, err = layoutArg(a[2])
if err != nil { return err
return err
}
if !namesADateField(layout) {
return fmt.Errorf("date(from,to,layout): '%s' names no date field; write time('%s')", layout, layout)
}
if from.Equal(to) && !namesAClockField(layout) {
return fmt.Errorf("date(%s,%s,'%s') is the constant %q; write it as text", a[0], a[1], layout, from.Format(layout))
}
return nil
} }
func timeArg(_ map[string]node, a []string) error { func timeArg(a []string) error {
if err := layoutArity("time", 1, a); err != nil { if err := layoutArity("time", 1, a); err != nil {
return err return err
} }
layout, err := layoutArg(a[0]) layout, err := layoutArg(a[0])
if err != nil { if err == nil && namesADateField(layout) {
return err return fmt.Errorf("time(layout): '%s' names a date field, and time draws no date; write date(from,to,layout)", layout)
} }
if namesADateField(layout) { return err
return fmt.Errorf("time(layout): '%s' names a date field; write date(from,to,layout)", layout) }
}
return nil // namesADateField reports a layout that prints two instants differing in their date alone apart.
func namesADateField(layout string) bool {
day, nextYear := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC), time.Date(2010, 11, 12, 4, 5, 6, 0, time.UTC)
return day.Format(layout) != nextYear.Format(layout)
} }
// datePrep draws a second in [from 00:00:00, to 23:59:59] UTC; the span is counted // datePrep draws a second in [from 00:00:00, to 23:59:59] UTC; the span is counted
// in seconds, since a Duration overflows past 292 years. // in seconds, since a Duration overflows past 292 years.
func datePrep(a []string) callFn { func datePrep(a []string) Call {
from, err := time.Parse(dayLayout, a[0]) from, err := time.Parse(dayLayout, a[0])
if err != nil { if err != nil {
panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", a[0], err)) panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", a[0], err))
} }
to, err := time.Parse(dayLayout, a[1]) to, err := time.Parse(dayLayout, a[1])
if err != nil { if err != nil {
panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", a[1], err)) panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", a[1], err))
} }
layout, start, span := layoutOf(a[2]), from.Unix(), int(to.Unix()-from.Unix())+86400 layout, start, span := layoutOf(a[2]), from.Unix(), int(to.Unix()-from.Unix())+86400
return func(s *session, _ string, _ []string) string { return func(s *drawstate.State, _ string, _ []string) string {
return time.Unix(start+int64(s.IntN(span)), 0).UTC().Format(layout) return time.Unix(start+int64(s.IntN(span)), 0).UTC().Format(layout)
} }
} }
func timePrep(a []string) callFn { func timePrep(a []string) Call {
layout := layoutOf(a[0]) layout := layoutOf(a[0])
return func(s *session, _ string, _ []string) string { return func(s *drawstate.State, _ string, _ []string) string {
return time.Unix(int64(s.IntN(86400)), 0).UTC().Format(layout) return time.Unix(int64(s.IntN(86400)), 0).UTC().Format(layout)
} }
} }
@@ -1,9 +1,13 @@
package fejkdata package builtinfunc
import ( import (
"fmt" "fmt"
"strings" "strings"
"unicode" "unicode"
"github.com/larvit/fejkdata/internal/drawstate"
"github.com/larvit/fejkdata/internal/grammar"
"github.com/larvit/fejkdata/internal/invariant"
) )
// transforms are the builtins that rewrite one operand's value; they nest, so // transforms are the builtins that rewrite one operand's value; they nest, so
@@ -14,11 +18,21 @@ var transforms = map[string]func(string) string{
"uppercase": strings.ToUpper, "uppercase": strings.ToUpper,
} }
func withTransforms(calls map[string]builtin) map[string]builtin {
for name, fn := range transforms {
if _, clash := calls[name]; clash {
panic(invariant.Broken("%s is registered as a builtin and a transform", name))
}
calls[name] = builtin{arity: 1, checkArgs: transformArg, prep: transformPrep(fn), operands: transformOperand}
}
return calls
}
// unwrapTransform peels nested transform calls off an operand arg, returning the // unwrapTransform peels nested transform calls off an operand arg, returning the
// field it finally names and the transforms to apply, innermost last. // field it finally names and the transforms to apply, innermost last.
func unwrapTransform(arg string) (leaf string, chain []func(string) string, err error) { func unwrapTransform(arg string) (leaf string, chain []func(string) string, err error) {
for { for {
name, args, isCall := funcCall(arg) name, args, isCall := grammar.FuncCall(arg)
if !isCall { if !isCall {
return arg, chain, nil return arg, chain, nil
} }
@@ -33,31 +47,29 @@ func unwrapTransform(arg string) (leaf string, chain []func(string) string, err
arg = args[0] arg = args[0]
} }
} }
func transformArg(fields map[string]node, a []string) error {
leaf, _, err := unwrapTransform(a[0]) // transformArg checks that each call nested in the arg is a transform taking one arg.
if err != nil { func transformArg(a []string) error {
return err _, _, err := unwrapTransform(a[0])
} return err
if isRef(leaf) {
_, _, err := refShape(leaf)
return err
}
return checkArm(leaf, fields, false)
} }
func transformOperand(a []string) []string { func transformOperand(a []string) []string {
if len(a) != 1 {
return nil
}
leaf, _, err := unwrapTransform(a[0]) leaf, _, err := unwrapTransform(a[0])
if err != nil { if err != nil {
return nil return nil
} }
return []string{leaf} return []string{leaf}
} }
func transformPrep(outer func(string) string) func([]string) callFn { func transformPrep(outer func(string) string) func([]string) Call {
return func(a []string) callFn { return func(a []string) Call {
_, chain, err := unwrapTransform(a[0]) _, chain, err := unwrapTransform(a[0])
if err != nil { if err != nil {
panic(fmt.Sprintf("fejkdata: transform arg %q reached prep unvalidated: %v", a[0], err)) panic(invariant.Broken("transform arg %q reached prep unvalidated: %v", a[0], err))
} }
return func(_ *session, _ string, operands []string) string { return func(_ *drawstate.State, _ string, operands []string) string {
v := operands[0] v := operands[0]
for i := len(chain) - 1; i >= 0; i-- { for i := len(chain) - 1; i >= 0; i-- {
v = chain[i](v) v = chain[i](v)
+215
View File
@@ -0,0 +1,215 @@
// Package datafiles walks a data tree and reads its category files and their rows.
package datafiles
import (
"errors"
"fmt"
"io/fs"
"os"
"path"
"strings"
"github.com/larvit/fejkdata/internal/grammar"
"github.com/larvit/fejkdata/internal/jsonvalue"
)
// Source is one tree to load, from the root of an fs.FS. label prefixes file names in
// errors; onDisk marks label as a directory that must exist; at is the folder fsys mounts the
// tree at, "" for its root.
type Source struct {
fsys fs.FS
label string
onDisk bool
at string
}
// Dir is the tree in the directory path, its errors labelled with it.
func Dir(path string) Source {
return Source{fsys: os.DirFS(path), label: path, onDisk: true}
}
// FS is the tree in fsys.
func FS(fsys fs.FS) Source { return Source{fsys: fsys} }
// Mounted is s with its tree under the folders the dot path namespace spells, "" for none.
func (s Source) Mounted(namespace string) Source {
if namespace != "" && s.at == "" {
s.at = strings.ReplaceAll(namespace, ".", "/")
s.fsys = mounted{fsys: s.fsys, at: s.at}
}
return s
}
// Category is one category file: the folders above it from the source's root, its name,
// and its parsed JSON.
type Category struct {
Folders []string
Name string
JSON jsonvalue.Value
rows *rowsFiles
}
// ReadRows reads a rows file beside the category.
func (c Category) ReadRows(name string) (string, error) { return c.rows.read(name) }
// labelled is the file p as errors name it: a labelled source's as its own tree spells it,
// below the mount.
func (s Source) labelled(p string) string {
if s.label == "" {
return p
}
if inner, below := underMount(s.at, p); below {
p = inner
}
return path.Join(s.label, p)
}
// dirPath is the folder that dir names, as fsys spells it.
func (s Source) dirPath(dir []string) string {
if p := path.Join(dir...); p != "" {
return p
}
return "."
}
// Walk hands compile every category of the tree, folders and files in name order. A
// folder holding no category anywhere below it is skipped; a hidden file or folder is no
// category, though a category may name a hidden rows file beside it.
func (s Source) Walk(compile func(Category) error) error {
if err := s.check(); err != nil {
return err
}
_, err := s.walkDir(nil, compile)
return err
}
// check refuses a data path that is empty or names no directory.
func (s Source) check() error {
if !s.onDisk {
return nil
}
if s.label == "" {
return fmt.Errorf("a data path is empty")
}
info, err := os.Stat(s.label)
if errors.Is(err, fs.ErrNotExist) {
return noDirectory(s.label)
}
if err != nil {
return fmt.Errorf("data path %s: %w", s.label, err)
}
if !info.IsDir() {
return fmt.Errorf("data path %s: not a directory", s.label)
}
return nil
}
// noDirectory is a data path naming nothing.
type noDirectory string
func (d noDirectory) Error() string { return "data path " + string(d) + ": no such directory" }
func (noDirectory) Unwrap() error { return fs.ErrNotExist }
// walkDir walks the folder that dir names, and reports whether it handed over any category.
func (s Source) walkDir(dir []string, compile func(Category) error) (bool, error) {
full := s.dirPath(dir)
entries, err := fs.ReadDir(s.fsys, full)
if err != nil {
return false, fmt.Errorf("%s: %w", s.labelled(full), err)
}
rows := newRowsFiles(s, full, entries)
handed := false
for _, e := range entries {
if strings.HasPrefix(e.Name(), ".") {
continue
}
var holds bool
if e.IsDir() {
holds, err = s.walkFolder(append(dir[:len(dir):len(dir)], e.Name()), compile)
} else {
holds, err = s.compileFile(dir, e.Name(), rows, compile)
}
if err != nil {
return false, err
}
handed = handed || holds
}
return handed, nil
}
// walkFolder walks a subfolder, and refuses its name if it holds a category.
func (s Source) walkFolder(dir []string, compile func(Category) error) (bool, error) {
holds, err := s.walkDir(dir, compile)
if err != nil || !holds {
return false, err
}
if err := grammar.CheckIdentifier(dir[len(dir)-1]); err != nil {
return false, fmt.Errorf("%s: folder %w", s.labelled(s.dirPath(dir)), err)
}
return true, nil
}
// Load hands compile the category file name.json in the folder dir.
func (s Source) Load(dir []string, name string, compile func(Category) error) error {
full := s.dirPath(dir)
entries, err := fs.ReadDir(s.fsys, full)
if err != nil {
return fmt.Errorf("%s: %w", s.labelled(full), err)
}
_, err = s.compileFile(dir, name+".json", newRowsFiles(s, full, entries), compile)
return err
}
// compileFile hands compile a *.json file as a category named after it, and reports
// whether it was one; any other file is skipped.
func (s Source) compileFile(dir []string, file string, rows *rowsFiles, compile func(Category) error) (bool, error) {
if !strings.HasSuffix(file, ".json") {
return false, nil
}
full := path.Join(rows.dir, file)
name := strings.TrimSuffix(file, ".json")
if err := grammar.CheckIdentifier(name); err != nil {
return false, fmt.Errorf("%s: category %w", s.labelled(full), err)
}
b, err := fs.ReadFile(s.fsys, full)
if err != nil {
return false, fmt.Errorf("%s: %w", s.labelled(full), err)
}
raw, err := jsonvalue.Decode(b)
if err != nil {
return false, fmt.Errorf("%s: %w", s.labelled(full), err)
}
if err := compile(Category{Folders: dir, Name: name, JSON: raw, rows: rows}); err != nil {
return false, fmt.Errorf("%s: %w", s.labelled(full), err)
}
return true, nil
}
// rowsFiles is what a category may name beside itself: the rows files of its folder.
type rowsFiles struct {
src Source
dir string
tsv map[string]bool
}
func newRowsFiles(src Source, dir string, entries []fs.DirEntry) *rowsFiles {
files := &rowsFiles{src: src, dir: dir, tsv: map[string]bool{}}
for _, e := range entries {
if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() {
files.tsv[e.Name()] = true
}
}
return files
}
func (r *rowsFiles) read(name string) (string, error) {
if !r.tsv[name] {
return "", fmt.Errorf("rows names %s, which is not beside it in %s", name, r.src.labelled(r.dir))
}
b, err := fs.ReadFile(r.src.fsys, path.Join(r.dir, name))
if err != nil {
return "", fmt.Errorf("%s: %w", r.src.labelled(path.Join(r.dir, name)), err)
}
return string(b), nil
}
+217
View File
@@ -0,0 +1,217 @@
package datafiles
import (
"errors"
"io/fs"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"testing/fstest"
)
type handed struct {
dir []string
name string
json any
}
func walk(t *testing.T, src Source) ([]handed, error) {
t.Helper()
var got []handed
err := src.Walk(func(c Category) error {
got = append(got, handed{c.Folders, c.Name, c.JSON.Any()})
return nil
})
return got, err
}
func TestWalkHandsEachCategoryInOrder(t *testing.T) {
got, err := walk(t, Source{fsys: fstest.MapFS{
"a.json": {Data: []byte(`{"x":"1"}`)},
"sub/b.json": {Data: []byte(`"y"`)},
"sub/notes.txt": {Data: []byte("not data")},
"empty/x.txt": {Data: []byte("not data")},
".git/c.json": {Data: []byte(`"hidden"`)},
"sub/.d.json": {Data: []byte(`"hidden"`)},
"sub/deep/e.json": {Data: []byte(`"z"`)},
}})
if err != nil {
t.Fatal(err)
}
want := []handed{
{nil, "a", map[string]any{"x": "1"}},
{[]string{"sub"}, "b", "y"},
{[]string{"sub", "deep"}, "e", "z"},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("Walk handed %#v, want %#v", got, want)
}
}
func TestReadRowsReadsOnlyAFileBesideTheCategory(t *testing.T) {
src := Source{fsys: fstest.MapFS{
"t.json": {Data: []byte(`{"rows":"t.tsv"}`)},
"t.tsv": {Data: []byte("key\nA\n")},
"sub/u.tsv": {Data: []byte("key\nB\n")},
"sub/u.json": {Data: []byte(`{"rows":"u.tsv"}`)},
}, label: "lbl"}
rows := map[string]string{}
err := src.Walk(func(c Category) error {
data, err := c.ReadRows(c.Name + ".tsv")
rows[c.Name] = data
if err != nil {
return err
}
if _, err := c.ReadRows("x.tsv"); err == nil || err.Error() != "rows names x.tsv, which is not beside it in "+map[string]string{"t": "lbl", "u": "lbl/sub"}[c.Name] {
t.Errorf("ReadRows(x.tsv) = %v, want an error saying x.tsv is not beside %s", err, c.Name)
}
return nil
})
if err != nil || rows["t"] != "key\nA\n" || rows["u"] != "key\nB\n" {
t.Errorf("Walk = %v, read %q", err, rows)
}
}
func TestWalkIgnoresARowsFileNoCategoryNames(t *testing.T) {
if _, err := walk(t, Source{fsys: fstest.MapFS{"a.json": {Data: []byte(`"x"`)}, "a.tsv": {Data: []byte("key\n")}}, label: "lbl"}); err != nil {
t.Errorf("Walk = %v, want a.tsv ignored", err)
}
}
func TestARowsFileMayBeHidden(t *testing.T) {
var rows string
err := Source{fsys: fstest.MapFS{"a.json": {Data: []byte(`"x"`)}, ".a.tsv": {Data: []byte("key\nA\n")}}}.Walk(func(c Category) error {
var err error
rows, err = c.ReadRows(".a.tsv")
return err
})
if err != nil || rows != "key\nA\n" {
t.Errorf("Walk = %v, read %q, want the hidden rows file read", err, rows)
}
}
func TestWalkLabelsWhatACategoryFileGetsWrong(t *testing.T) {
for _, c := range []struct {
name string
fs fstest.MapFS
compile error
want string
}{
{"its compile", fstest.MapFS{"a.json": {Data: []byte(`"x"`)}}, errors.New("bad"), "lbl/a.json: bad"},
{"its JSON", fstest.MapFS{"a.json": {Data: []byte(`{`)}}, nil, "lbl/a.json: line 1, column 1: unexpected end of JSON input"},
{"its name", fstest.MapFS{"a.b.json": {Data: []byte(`"x"`)}}, nil, "lbl/a.b.json: category "},
{"its folder's name", fstest.MapFS{"a.b/c.json": {Data: []byte(`"x"`)}}, nil, "lbl/a.b: folder "},
} {
err := Source{fsys: c.fs, label: "lbl"}.Walk(func(Category) error { return c.compile })
if err == nil || !strings.HasPrefix(err.Error(), c.want) {
t.Errorf("%s: Walk = %v, want it to start %q", c.name, err, c.want)
}
}
}
func TestWalkChecksAFolderNameOnlyWhenItHoldsData(t *testing.T) {
if _, err := walk(t, Source{fsys: fstest.MapFS{"a.b/notes.txt": {Data: []byte("x")}, "c.json": {Data: []byte(`"x"`)}}}); err != nil {
t.Errorf("Walk = %v, want a folder holding no data skipped whatever its name", err)
}
}
func TestWalkRefusesADiskPathThatIsNoDirectory(t *testing.T) {
dir := t.TempDir()
for _, c := range []struct{ path, want string }{
{"", "a data path is empty"},
{filepath.Join(dir, "none"), "data path " + filepath.Join(dir, "none") + ": no such directory"},
} {
_, err := walk(t, Dir(c.path))
if err == nil || err.Error() != c.want {
t.Errorf("Walk(%q) = %v, want %q", c.path, err, c.want)
}
}
}
func TestLoadHandsOneCategoryAndItsRows(t *testing.T) {
src := Source{fsys: fstest.MapFS{
"sub/t.json": {Data: []byte(`{"rows":"t.tsv"}`)},
"sub/t.tsv": {Data: []byte("key\nA\n")},
"sub/u.json": {Data: []byte(`{`)},
}}
var got handed
var rows string
err := src.Load([]string{"sub"}, "t", func(c Category) error {
got = handed{c.Folders, c.Name, c.JSON.Any()}
var err error
rows, err = c.ReadRows("t.tsv")
return err
})
if err != nil || got.name != "t" || !reflect.DeepEqual(got.dir, []string{"sub"}) || rows != "key\nA\n" {
t.Errorf("Load = %v, handed %#v with rows %q", err, got, rows)
}
}
func TestDirReadsTheDirectoryAndLabelsItsErrorsWithIt(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "a.json"), []byte(`{`), 0o600); err != nil {
t.Fatal(err)
}
_, err := walk(t, Dir(dir))
if err == nil || !strings.HasPrefix(err.Error(), filepath.ToSlash(filepath.Join(dir, "a.json"))+": ") {
t.Errorf("Walk = %v, want the error labelled with the directory", err)
}
if _, err := walk(t, Dir(filepath.Join(dir, "a.json"))); err == nil || err.Error() != "data path "+filepath.Join(dir, "a.json")+": not a directory" {
t.Errorf("Walk = %v, want a file refused as no directory", err)
}
}
func TestAMountedDirNamesItsFilesWhereTheyAreOnDisk(t *testing.T) {
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, "sub"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "sub", "a.json"), []byte(`{`), 0o600); err != nil {
t.Fatal(err)
}
_, err := walk(t, Dir(dir).Mounted("my.data"))
if err == nil || !strings.HasPrefix(err.Error(), filepath.ToSlash(filepath.Join(dir, "sub", "a.json"))+": ") {
t.Errorf("Walk = %v, want the error to name the file on disk", err)
}
}
func TestAMountedTreeListsAndReadsUnderItsMount(t *testing.T) {
m := mounted{fsys: fstest.MapFS{"a.json": {Data: []byte(`"x"`)}, "sub/b.json": {Data: []byte(`"y"`)}}, at: "my/data"}
for dir, want := range map[string][]string{".": {"my"}, "my": {"data"}, "my/data": {"a.json", "sub"}, "my/data/sub": {"b.json"}} {
entries, err := fs.ReadDir(m, dir)
var got []string
for _, e := range entries {
got = append(got, e.Name())
}
if err != nil || !reflect.DeepEqual(got, want) {
t.Errorf("ReadDir(%q) = %v, %v, want %v", dir, got, err, want)
}
}
if b, err := fs.ReadFile(m, "my/data/sub/b.json"); err != nil || string(b) != `"y"` {
t.Errorf(`ReadFile("my/data/sub/b.json") = %q, %v, want "y"`, b, err)
}
if _, err := fs.ReadFile(m, "other/a.json"); !errors.Is(err, fs.ErrNotExist) {
t.Errorf(`ReadFile("other/a.json") = %v, want it not to exist`, err)
}
}
func TestAMountedSourceNamesItsManifestBesideItsFiles(t *testing.T) {
dir := t.TempDir()
for name, c := range map[string]struct {
src Source
want string
}{
"an fs.FS": {FS(fstest.MapFS{}).Mounted("my.data"), "my/data/.fejkdata.json"},
"a data path": {Dir(dir).Mounted("my.data"), filepath.ToSlash(filepath.Join(dir, ".fejkdata.json"))},
} {
if got := c.src.ManifestPath(); got != c.want {
t.Errorf("%s: ManifestPath() = %q, want %q", name, got, c.want)
}
}
m, err := FS(fstest.MapFS{ManifestFile: {Data: []byte(`{"namespace": "my.data"}`)}}).Mounted("my.data").Manifest()
if err != nil || m.Namespace != "my.data" {
t.Errorf("Manifest() after Mounted = %+v, %v, want the manifest read from beside the files", m, err)
}
}
+246
View File
@@ -0,0 +1,246 @@
package datafiles
import (
"errors"
"fmt"
"io/fs"
"path"
"sort"
"strings"
"github.com/larvit/fejkdata/internal/grammar"
"github.com/larvit/fejkdata/internal/jsonvalue"
)
// ManifestFile is the file at a source's root that describes the source.
const ManifestFile = ".fejkdata.json"
// Manifest is what a source's manifest says of it.
type Manifest struct {
// Index is every category of the source by dot path; nil where the manifest has none.
Index map[string]IndexEntry
// Namespace is the dot path the source's root sits under, "" for the namespace's root.
Namespace string
}
// IndexEntry is what an index says of a category: its parent table, "" for none, and the
// paths List advertises below it.
type IndexEntry struct {
Parent string `json:"parent,omitempty"`
Paths []string `json:"paths"`
}
// Manifest reads the manifest at the source's root, the zero Manifest where there is none.
func (s Source) Manifest() (Manifest, error) {
if err := s.check(); err != nil {
return Manifest{}, err
}
b, err := fs.ReadFile(s.fsys, path.Join(s.at, ManifestFile))
if errors.Is(err, fs.ErrNotExist) {
return Manifest{}, nil
}
if err != nil {
return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err)
}
m, err := decodeManifest(b)
if err != nil {
return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err)
}
return m, nil
}
// ManifestPath is the manifest's path as errors name it.
func (s Source) ManifestPath() string { return s.labelled(path.Join(s.at, ManifestFile)) }
// decodeManifest decodes a manifest, refusing any key or shape it does not define, and
// reports every mistake: first each entry's own, in path order, then each parent naming
// no entry.
func decodeManifest(b []byte) (Manifest, error) {
raw, err := jsonvalue.Decode(b)
if err != nil {
return Manifest{}, err
}
top, isObject := raw.Any().(map[string]any)
if !isObject {
return Manifest{}, fmt.Errorf("a manifest must be an object, not %s", jsonvalue.Kind(raw.Any()))
}
var m Manifest
var errs []error
for _, k := range sortedKeys(top) {
switch k {
case "index":
var indexErrs []error
m.Index, indexErrs = decodeIndex(top[k])
errs = append(errs, indexErrs...)
case "namespace":
var err error
m.Namespace, err = decodeNamespace(top[k])
errs = append(errs, err)
default:
errs = append(errs, fmt.Errorf("unknown key %q; a manifest holds index and namespace", k))
}
}
for _, p := range sortedKeys(m.Index) {
if m.Namespace != "" && !strings.HasPrefix(p, m.Namespace+".") {
errs = append(errs, fmt.Errorf("index entry %q lies outside the namespace %q", p, m.Namespace))
}
}
return m, errors.Join(errs...)
}
func decodeNamespace(v any) (string, error) {
namespace, isString := v.(string)
if !isString {
return "", fmt.Errorf("namespace must be a string, not %s", jsonvalue.Kind(v))
}
if namespace == "" {
return "", nil
}
if err := checkCategoryPath(namespace); err != nil {
return "", fmt.Errorf("namespace %w", err)
}
return namespace, nil
}
func decodeIndex(v any) (map[string]IndexEntry, []error) {
entries, isObject := v.(map[string]any)
if !isObject {
return nil, []error{fmt.Errorf("index must be an object, not %s", jsonvalue.Kind(v))}
}
index := map[string]IndexEntry{}
var errs []error
for _, p := range sortedKeys(entries) {
e, err := decodeEntry(p, entries[p])
if err != nil {
errs = append(errs, fmt.Errorf("index entry %q: %w", p, err))
}
index[p] = e
}
for _, p := range sortedKeys(index) {
for _, err := range []error{checkOutside(index, p), checkParent(index, p)} {
if err != nil {
errs = append(errs, fmt.Errorf("index entry %q: %w", p, err))
}
}
}
return index, errs
}
// decodeEntry decodes one entry, leaving out a parent it refuses, so checkParent names no
// mistake twice.
func decodeEntry(p string, v any) (IndexEntry, error) {
var e IndexEntry
errs := []error{checkCategoryPath(p)}
fields, isObject := v.(map[string]any)
if !isObject {
errs = append(errs, fmt.Errorf("an entry must be an object, not %s", jsonvalue.Kind(v)))
return e, errors.Join(errs...)
}
if _, has := fields["paths"]; !has {
errs = append(errs, fmt.Errorf("paths is missing"))
}
for _, k := range sortedKeys(fields) {
switch k {
case "parent":
parent, err := decodeParent(fields[k])
e.Parent = parent
errs = append(errs, err)
case "paths":
var err error
e.Paths, err = decodePaths(fields[k])
errs = append(errs, err)
default:
errs = append(errs, fmt.Errorf("unknown key %q; an entry holds parent and paths", k))
}
}
return e, errors.Join(errs...)
}
func decodeParent(v any) (string, error) {
parent, isString := v.(string)
if !isString {
return "", fmt.Errorf("parent must be a string, not %s", jsonvalue.Kind(v))
}
if err := grammar.CheckIdentifier(parent); err != nil {
return "", fmt.Errorf("parent %w", err)
}
return parent, nil
}
// decodePaths decodes a list of paths below a category, "" being the category itself.
func decodePaths(v any) ([]string, error) {
items, isList := v.([]any)
if !isList {
return nil, fmt.Errorf("paths must be a list, not %s", jsonvalue.Kind(v))
}
out := make([]string, 0, len(items))
seen := map[string]bool{}
var errs []error
for i, item := range items {
q, isString := item.(string)
if !isString {
errs = append(errs, fmt.Errorf("paths item %d must be a string, not %s", i+1, jsonvalue.Kind(item)))
continue
}
if q != "" {
if err := checkCategoryPath(q); err != nil {
errs = append(errs, fmt.Errorf("paths item %d: %w", i+1, err))
continue
}
}
if seen[q] {
errs = append(errs, fmt.Errorf("paths item %d repeats %q", i+1, q))
continue
}
seen[q] = true
out = append(out, q)
}
return out, errors.Join(errs...)
}
// checkOutside refuses an entry whose path passes through another entry's category.
func checkOutside(index map[string]IndexEntry, p string) error {
segs := strings.Split(p, ".")
for n := 1; n < len(segs); n++ {
outer := strings.Join(segs[:n], ".")
if _, isEntry := index[outer]; isEntry {
return fmt.Errorf("%q is a category, so it holds no other", outer)
}
}
return nil
}
// checkParent refuses a parent naming no entry beside p: a table loads with its parent,
// from one source.
func checkParent(index map[string]IndexEntry, p string) error {
parent := index[p].Parent
if parent == "" {
return nil
}
sibling := parent
if dot := strings.LastIndexByte(p, '.'); dot >= 0 {
sibling = p[:dot+1] + parent
}
if _, ok := index[sibling]; !ok {
return fmt.Errorf("parent %q names no entry in this entry's folder; a table in an indexed source links only to a parent in the same index, so to link under another source's table, drop this index, and the source loads whole at start", parent)
}
return nil
}
func checkCategoryPath(p string) error {
for _, seg := range strings.Split(p, ".") {
if err := grammar.CheckIdentifier(seg); err != nil {
return fmt.Errorf("%q: %w", p, err)
}
}
return nil
}
func sortedKeys[V any](m map[string]V) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
+73
View File
@@ -0,0 +1,73 @@
package datafiles
import (
"io/fs"
"strings"
)
// mounted is fsys with its root moved down to the folder at, so a file f of fsys is at/f. It
// serves what a walk asks of a tree, a folder's entries and a file's bytes: the folders above
// at list only the next folder towards it, and opening one fails.
type mounted struct {
fsys fs.FS
at string
}
func (m mounted) Open(name string) (fs.File, error) {
if inner, below := m.inner(name); below {
return m.fsys.Open(inner)
}
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
}
func (m mounted) ReadFile(name string) ([]byte, error) {
if inner, below := m.inner(name); below {
return fs.ReadFile(m.fsys, inner)
}
return nil, &fs.PathError{Op: "read", Path: name, Err: fs.ErrNotExist}
}
func (m mounted) ReadDir(name string) ([]fs.DirEntry, error) {
if inner, below := m.inner(name); below {
return fs.ReadDir(m.fsys, inner)
}
rest, above := strings.CutPrefix(m.at, name+"/")
if name == "." {
rest, above = m.at, true
}
if !above {
return nil, &fs.PathError{Op: "readdir", Path: name, Err: fs.ErrNotExist}
}
next, _, _ := strings.Cut(rest, "/")
return []fs.DirEntry{folderEntry(next)}, nil
}
// inner is name as fsys spells it, where name lies at or below at.
func (m mounted) inner(name string) (string, bool) {
inner, below := underMount(m.at, name)
return inner, below && fs.ValidPath(name)
}
// underMount is name as the tree mounted at at spells it, where name lies at or below at.
func underMount(at, name string) (string, bool) {
switch name {
case at:
return ".", true
case "":
return name, false
}
if at == "" {
return name, true
}
return strings.CutPrefix(name, at+"/")
}
// folderEntry is a folder above the mount, as a folder's entries list it.
type folderEntry string
func (e folderEntry) Name() string { return string(e) }
func (e folderEntry) IsDir() bool { return true }
func (e folderEntry) Type() fs.FileMode { return fs.ModeDir }
func (e folderEntry) Info() (fs.FileInfo, error) {
return nil, &fs.PathError{Op: "stat", Path: string(e), Err: fs.ErrNotExist}
}
+39
View File
@@ -0,0 +1,39 @@
// Package datatype names what a record column holds.
package datatype
import "fmt"
// DataType is what a record column holds, which decides how a record writes its value.
type DataType int
// The datatypes a column declares with "datatype"; a column without one is a string.
const (
String DataType = iota
Integer
Number
Boolean
)
// Count is the number of datatypes, for an array indexed by one.
const Count = Boolean + 1
var (
names = [Count]string{"string", "integer", "number", "boolean"}
nouns = [Count]string{"text", "an integer", "a number", "a boolean"}
)
// String is the datatype as data spells it.
func (d DataType) String() string {
if d < 0 || d >= Count {
return fmt.Sprintf("DataType(%d)", int(d))
}
return names[d]
}
// Noun is the datatype as an error names what a value is.
func Noun(d DataType) string {
if d < 0 || d >= Count {
return d.String()
}
return nouns[d]
}
+31
View File
@@ -0,0 +1,31 @@
package datatype
import "testing"
func TestEachDatatypeIsSpelledAsDataWritesIt(t *testing.T) {
for d, want := range map[DataType][2]string{
String: {"string", "text"},
Integer: {"integer", "an integer"},
Number: {"number", "a number"},
Boolean: {"boolean", "a boolean"},
} {
if d.String() != want[0] || Noun(d) != want[1] {
t.Errorf("%d: String = %q, Noun = %q, want %q", int(d), d.String(), Noun(d), want)
}
}
if Count != 4 {
t.Errorf("Count = %d, want the four datatypes", Count)
}
}
func TestStringNamesADatatypeNoDataWrites(t *testing.T) {
if got := DataType(9).String(); got != "DataType(9)" {
t.Errorf("String = %q, want DataType(9)", got)
}
}
func TestNounNamesADatatypeNoDataWrites(t *testing.T) {
if got := Noun(DataType(9)); got != "DataType(9)" {
t.Errorf("Noun = %q, want DataType(9)", got)
}
}
+37
View File
@@ -0,0 +1,37 @@
// Package drawstate holds a generator's seeded randomness and its {seq()} counters.
package drawstate
import (
"math/rand/v2"
"sort"
)
// State is one generator's draw state, kept across its renders.
type State struct {
rand *rand.Rand
counters map[string]uint64
}
// New builds the state a seed reproduces.
func New(seed uint64) *State {
return &State{rand: rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15)), counters: map[string]uint64{}}
}
func (s *State) IntN(n int) int { return s.rand.IntN(n) }
func (s *State) Float64() float64 { return s.rand.Float64() }
// Weighted draws an index into cum, a running sum of weights of 0 or more with a positive total,
// with odds by weight, so an index of weight 0 is never drawn.
func (s *State) Weighted(cum []float64) int {
total := cum[len(cum)-1]
x := s.Float64() * total
// x stays below total: a product of a float64 under 1 and a positive float64 rounds below it.
return sort.Search(len(cum), func(i int) bool { return cum[i] > x })
}
// Seq advances the counter {seq(key)} reads, or {seq()} for an empty key, and returns it.
func (s *State) Seq(key string) uint64 {
s.counters[key]++
return s.counters[key]
}

Some files were not shown because too many files have changed in this diff Show More