Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 lilleman
|
4c8f30406e |
@@ -1,162 +0,0 @@
|
||||
name: Packages
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
tag:
|
||||
type: string
|
||||
required: true
|
||||
secrets:
|
||||
TAP_GITHUB_TOKEN:
|
||||
required: true
|
||||
PACKAGES_GPG_KEY:
|
||||
required: true
|
||||
PACKAGES_RSA_KEY:
|
||||
required: true
|
||||
DOCKERHUB_TOKEN:
|
||||
required: true
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: The released tag to package, such as v0.1.0
|
||||
type: string
|
||||
required: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: packages
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
packages:
|
||||
name: Binaries, packages and image for ${{ inputs.tag }}
|
||||
runs-on: ubuntu-26.04
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
outputs:
|
||||
latest: ${{ steps.latest.outputs.latest }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ inputs.tag }}
|
||||
path: src
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
# Check out the packaging config at the workflow's commit, so a tag cut before the
|
||||
# config existed can still be packaged.
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
path: tooling
|
||||
persist-credentials: false
|
||||
- name: The Go version the gate tests
|
||||
id: go
|
||||
run: sed -n 's/^ARG GO_VERSION=/version=/p' tooling/Dockerfile >> "$GITHUB_OUTPUT"
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ steps.go.outputs.version }}
|
||||
cache: false
|
||||
- name: Channels holding one version
|
||||
id: latest
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ inputs.tag }}
|
||||
run: |
|
||||
latest="$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName --jq .tagName)"
|
||||
if [ "$latest" = "$TAG" ]; then
|
||||
echo "latest=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "skip=--skip=homebrew,scoop" >> "$GITHUB_OUTPUT"
|
||||
echo "OLDER_TAG=1" >> "$GITHUB_ENV"
|
||||
fi
|
||||
- uses: docker/setup-buildx-action@v4
|
||||
- uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- uses: goreleaser/goreleaser-action@v7
|
||||
with:
|
||||
version: v2.18.3
|
||||
workdir: src
|
||||
args: release --clean --config ../tooling/release-tooling/goreleaser.yaml ${{ steps.latest.outputs.skip }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }}
|
||||
- name: The built binary and the published image print the tag
|
||||
env:
|
||||
TAG: ${{ inputs.tag }}
|
||||
run: |
|
||||
test "$(dist/fejkdata_linux_amd64_v1/fejkdata --version)" = "fejkdata $TAG"
|
||||
test "$(docker run --rm "ghcr.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG"
|
||||
|
||||
dockerhub:
|
||||
name: Docker Hub image for ${{ inputs.tag }}
|
||||
needs: packages
|
||||
runs-on: ubuntu-26.04
|
||||
permissions:
|
||||
packages: read
|
||||
steps:
|
||||
- uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ vars.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Copy the GHCR image to Docker Hub, and check it prints the tag
|
||||
env:
|
||||
LATEST: ${{ needs.packages.outputs.latest }}
|
||||
TAG: ${{ inputs.tag }}
|
||||
run: |
|
||||
tags=(-t "docker.io/larvit/fejkdata:${TAG#v}")
|
||||
if [ "$LATEST" = true ]; then
|
||||
tags+=(-t docker.io/larvit/fejkdata:latest)
|
||||
fi
|
||||
docker buildx imagetools create "${tags[@]}" "ghcr.io/larvit/fejkdata:${TAG#v}"
|
||||
test "$(docker run --rm "docker.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG"
|
||||
|
||||
repos:
|
||||
name: apt, dnf and apk repositories for ${{ inputs.tag }}
|
||||
needs: packages
|
||||
if: needs.packages.outputs.latest == 'true'
|
||||
runs-on: ubuntu-26.04
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deploy.outputs.page_url }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: The release's Linux packages
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ inputs.tag }}
|
||||
run: gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --dir packages --pattern '*.deb' --pattern '*.rpm' --pattern '*.apk'
|
||||
- uses: docker/setup-buildx-action@v4
|
||||
- name: Install fejkdata from each repository, then export them
|
||||
env:
|
||||
PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }}
|
||||
PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }}
|
||||
TAG: ${{ inputs.tag }}
|
||||
run: |
|
||||
build() {
|
||||
docker buildx build -f release-tooling/package-repos.Dockerfile \
|
||||
--secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY \
|
||||
--build-context packages=packages --build-arg "EXPECTED_VERSION=$TAG" "$@" .
|
||||
}
|
||||
build --target checks --output type=local,dest=checks
|
||||
build --target site --output type=local,dest=site
|
||||
- uses: actions/upload-pages-artifact@v5
|
||||
with:
|
||||
path: site
|
||||
- id: deploy
|
||||
uses: actions/deploy-pages@v5
|
||||
+4
-117
@@ -4,9 +4,6 @@ on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '17 5 * * *'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -14,127 +11,17 @@ permissions:
|
||||
jobs:
|
||||
test:
|
||||
name: vet + fmt + tests
|
||||
if: github.event_name != 'schedule'
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
- name: Relevant changes
|
||||
id: changes
|
||||
run: |
|
||||
if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
|
||||
git fetch --no-tags origin "$GITHUB_BASE_REF"
|
||||
base="origin/$GITHUB_BASE_REF"
|
||||
else
|
||||
base='${{ github.event.before }}'
|
||||
fi
|
||||
if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum Dockerfile .dockerignore .github data release-tooling testdata vulnerability-filing README.md todo.md; then
|
||||
echo "code=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "code=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum release-tooling Dockerfile .dockerignore LICENSE DATA-LICENSES.md .github/workflows/packages.yml .github/workflows/test.yml; then
|
||||
echo "packaging=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "packaging=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
echo "base=$base" >> "$GITHUB_OUTPUT"
|
||||
- name: CHANGELOG entry for data changes
|
||||
run: |
|
||||
base='${{ steps.changes.outputs.base }}'
|
||||
if ! git diff --quiet "$base"...HEAD -- data testdata/shipped_shape.txt && git diff --quiet "$base"...HEAD -- CHANGELOG.md; then
|
||||
echo "::error::the shipped data changed without a CHANGELOG.md entry (see the README's Versioning)"
|
||||
exit 1
|
||||
fi
|
||||
# `docker build` streams the context to the daemon. A compose bind-mount of
|
||||
# `.` mounts an empty host dir instead, because the job is itself a container.
|
||||
- name: Latest supported Go
|
||||
if: ${{ steps.changes.outputs.code == 'true' }}
|
||||
run: docker build .
|
||||
# Runs even when the step above failed, so a red build says whether the
|
||||
# failure is version-specific.
|
||||
- name: Lowest supported Go
|
||||
if: ${{ !cancelled() && steps.changes.outputs.code == 'true' }}
|
||||
run: docker build --build-arg GO_VERSION=1.22.12 --target portable .
|
||||
- name: Package repositories, signed with throwaway keys
|
||||
if: ${{ !cancelled() && steps.changes.outputs.packaging == 'true' }}
|
||||
run: |
|
||||
docker build -f release-tooling/package-repos.Dockerfile --target throwaway-keys --output "$RUNNER_TEMP/keys" .
|
||||
PACKAGES_GPG_KEY="$(cat "$RUNNER_TEMP/keys/gpg.asc")" PACKAGES_RSA_KEY="$(cat "$RUNNER_TEMP/keys/rsa.pem")" \
|
||||
docker build -f release-tooling/package-repos.Dockerfile --target checks \
|
||||
--secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY .
|
||||
|
||||
vulnerabilities:
|
||||
name: Known vulnerabilities in this Go version and the required modules
|
||||
runs-on: ubuntu-26.04
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
actions: write
|
||||
contents: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Fetch govulncheck's report from vuln.go.dev
|
||||
run: docker build --no-cache-filter vulnerabilities --target vulnerability-report --output "$RUNNER_TEMP/vulnerabilities" .
|
||||
- name: File each one in todo.md
|
||||
id: file
|
||||
run: python3 vulnerability-filing/file_vulnerabilities.py "$RUNNER_TEMP/vulnerabilities/report.json" todo.md
|
||||
- name: Propose the filed items
|
||||
if: failure() && steps.file.outcome == 'failure' && github.ref == 'refs/heads/main'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
if git diff --quiet todo.md || [ "$(gh pr list --head vulnerabilities --state open --json number --jq length)" != 0 ]; then
|
||||
exit 0
|
||||
fi
|
||||
git config user.name 'github-actions[bot]'
|
||||
git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
|
||||
git switch -C vulnerabilities
|
||||
git commit -qam "File the known vulnerabilities govulncheck reports"
|
||||
git push -qf "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" vulnerabilities
|
||||
gh pr create --head vulnerabilities --title "File the known vulnerabilities govulncheck reports" \
|
||||
--body "govulncheck reports known vulnerabilities in the Go version or a required module, and no release is cut until they are gone. This files them atop the next release in todo.md, scored 0.0 until rank_todo.py scores them; each item names the bump that fixes it."
|
||||
# A pull request the workflow's token opens starts no workflow.
|
||||
gh workflow run test.yml --ref vulnerabilities
|
||||
|
||||
release:
|
||||
name: Release from CHANGELOG.md
|
||||
needs: [test, vulnerabilities]
|
||||
if: github.event_name == 'push'
|
||||
runs-on: ubuntu-26.04
|
||||
permissions:
|
||||
contents: write
|
||||
outputs:
|
||||
tag: ${{ steps.publish.outputs.tag }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- id: publish
|
||||
run: python3 release-tooling/publish_release.py
|
||||
env:
|
||||
FORGE_API_URL: ${{ github.api_url }}
|
||||
FORGE_REPOSITORY: ${{ github.repository }}
|
||||
FORGE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SHA: ${{ github.sha }}
|
||||
|
||||
packages:
|
||||
needs: release
|
||||
if: needs.release.outputs.tag != ''
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
pages: write
|
||||
id-token: write
|
||||
uses: ./.github/workflows/packages.yml
|
||||
with:
|
||||
tag: ${{ needs.release.outputs.tag }}
|
||||
secrets:
|
||||
TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }}
|
||||
PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }}
|
||||
PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }}
|
||||
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
if: ${{ !cancelled() }}
|
||||
run: docker build --build-arg GO_VERSION=1.22.12 .
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
.claude
|
||||
*.out
|
||||
__pycache__/
|
||||
data-import/cache/
|
||||
todo.md
|
||||
@@ -1,137 +1,18 @@
|
||||
# Rules
|
||||
|
||||
- Go runs only through `docker compose run --rm <test|vet|fmt|build|cyclo|generate|vulncheck>`; the merge gate is what the README's Development section lists.
|
||||
- Tests first, in their own commit; the implementation follows in the next. A re-pin of seeded output or of `testdata/shipped_shape.txt` is its own commit.
|
||||
- A change under `data/` or to the shape pin adds its `CHANGELOG.md` entry under `Unreleased` in the same PR, and so does a change to a flag, an exit code, an exported name, a fence, a builtin or the lowest Go; what is major is the README's Versioning table.
|
||||
- Go runs only through `docker compose run --rm <test|vet|fmt|build>`; the merge gate is `docker build .`.
|
||||
- Tests first, in their own commit; the implementation follows in the next. A re-pin of seeded output is its own commit.
|
||||
- One-line commit messages: no ticket prefix, no repo name, no authorship trailers.
|
||||
- GitHub is where the project lives, so `gh` is the tool: merges are fast-forward
|
||||
only, `gh pr merge --rebase` where the branch is already on top of main.
|
||||
- CodeRabbit skips every pull request until the repository has 10 GitHub stars: check
|
||||
its status once when a PR is marked ready, answer a review if one is there, and never
|
||||
wait for one.
|
||||
- From 0.2.0, README goal 3 is gated at 7.0 on the `comprehension-panel` skill, and below it
|
||||
nothing else merges — no feature, no category, no data — bar a security fix, a
|
||||
dependency bump, a repair of behaviour the README documents, the infrastructure
|
||||
the round itself runs on, and a chunk the comprehension-floor decision lists as
|
||||
shipped under the floor. `todo.md` carries the round: a PR per item it plans, the nine-seat
|
||||
findings at depth 1 or a restructure the maintainer approved, then the nine-seat run
|
||||
again, until the score passes. At or above 7.0
|
||||
every pull request scores with the four-seat run and answers it in one run. No merge
|
||||
lowers the last score its own panel recorded: the nine-seat round and the four-seat
|
||||
run are two ratchets, never compared with each other. A lower score blocks the merge
|
||||
even where it may be panel noise: fix what the seats name until the score rises. A PR
|
||||
that only records a run's scores and files what its seats name merges even when a
|
||||
score fell. Through 0.1.0, a merge is blocked only by a paired ruling that rules worse.
|
||||
- Hard tabs. No comment by default; delete a restatement, a rationale, history, or a file preamble.
|
||||
- Prose naming a source states what that source states: read the register's own field
|
||||
before paraphrasing it, and name the register you read before claiming none
|
||||
publishes a spelling.
|
||||
- One spelling per result: reject the other at `New`, and let the error name the spelling to use.
|
||||
- A standing choice a reader would relitigate goes under Decisions in the README, not in a comment.
|
||||
- A README example is a `json` block that loads and renders as a category; `readme_test.go` runs every one.
|
||||
- Cyclomatic complexity is gated at 14: split a function that would go over it. A table-shaped dispatch, a switch with one case per kind of its input, may sit at 14 and stays whole.
|
||||
- Cyclomatic complexity is gated at 14: the table-shaped dispatches (`eachToken`, `calc.factor`, `walkPath`) sit at 13–14 and stay whole; anything else that reaches 14 is decomposed.
|
||||
|
||||
# Decisions
|
||||
# Deferred
|
||||
|
||||
In [docs/decisions.md](docs/decisions.md):
|
||||
|
||||
- GitHub is canonical, and the module path names it
|
||||
- The Gitea copy stays, as a pull mirror of GitHub
|
||||
- The vocabulary sits below `doc.go`'s package clause, not in the package doc
|
||||
- Options and fields share one namespace
|
||||
- `{a|b}` stays beside nested choices
|
||||
- Flags follow getopt_long
|
||||
- A struct tag is a template by its shape
|
||||
- The CLI renders exactly the template on its stdin, and adds nothing
|
||||
- The CLI reads only the library's public API
|
||||
- A template that does not compile is misuse (exit 2), including a reference that resolves to nothing, save a lone reference under the CLI's `--format` until item 129
|
||||
- Whitespace around a JSON object, array, string or null is dropped, and a bare number or boolean is a format string
|
||||
- `FakeTemplate` and `NewTemplate` both stay
|
||||
- The shipped data is embedded, not discovered
|
||||
- The shipped data is Go packages a library imports by choice, and the CLI carries every one, from 0.1.0
|
||||
- A shipped package's folder spells its namespace, its Go name is that namespace in lower case without separators, and its manifest mounts it with `namespace`
|
||||
- A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads
|
||||
- Every `{…}` draws afresh, and only a name keeps a pick
|
||||
- Reference sigils follow the filesystem
|
||||
- A change to what exists is a major; a minor only adds
|
||||
- An error is a contract by what it names, not its bytes
|
||||
- A load reports every mistake at once
|
||||
- Formats are checked after all data is loaded, so their errors name the category
|
||||
- Raising the lowest supported Go is a major
|
||||
- The format check runs on the latest Go only
|
||||
- The changelog heading is the one spelling of a release; CI cuts the tag
|
||||
- A `--data-path` override rebinds every reference to the category it replaces
|
||||
- The repeat cap bounds renders, not bytes
|
||||
- 64-bit targets only
|
||||
- A constant zero divisor is a load error; in a string column a divisor that is not constant prints `Inf`
|
||||
- Samples say what they emit, transforms what they do
|
||||
- A record is a template seen as columns; a Go struct is the one second schema
|
||||
- A struct's records follow Go's field access, and compile on first use
|
||||
- A category may reference itself, and a record's fences run at load
|
||||
- A record's column set is fixed before the first draw
|
||||
- Null is a `null` item, not a rate
|
||||
- A typed column holds one value, not composed text
|
||||
- A column that only reads one reference or name is the column it reads
|
||||
- A typed column's calc is refused unless proven
|
||||
- `Column` carries text, not a Go value
|
||||
- A unit takes the stem of what it is, and a file the stem of the units it holds
|
||||
- One name, one meaning
|
||||
- The performance gate asserts allocations, not wall-clock time
|
||||
- Rows live in a TSV, the shape in JSON
|
||||
- A selector is bracketed, and a dot inside it is literal
|
||||
- `parent` names the link column and the table alike
|
||||
- After a row, a path names a column or a linked table
|
||||
- A selected row is fixed, not drawn
|
||||
- `..` steps up to the parent row, and a step down after it draws afresh, or once per name's pick
|
||||
- A table selects by one key column, a code, never a free-form name
|
||||
- A parent row with no child row is a load error
|
||||
- A table is a record of string columns
|
||||
- The key map is built at load, the rest on first draw
|
||||
- Two categories may name one TSV
|
||||
- A cell may read a reference, and each row its own
|
||||
- A path is walked once without drawing before it is walked for real
|
||||
- A country's postal codes and streets are siblings under its locality
|
||||
- A locale package imports its country's geo package, and its `Modules()` returns both
|
||||
- The default embed holds every Swedish postort the import can place, with all its streets, and, of the US places that hold most of a postal code's land, every state capital, every place of 100,000 or more and a population-weighted tenth, each with a quarter of its streets, at least ten
|
||||
- A locale's `address` binds its country's record once and prints it whole
|
||||
- A postort's kommun comes from its name, its tätort or its codes, never from distance
|
||||
- A highway designation is not a street, and a US postal code belongs to the place holding most of its land inside places
|
||||
- `--list` stays a plain list of paths
|
||||
- A layout is always quoted
|
||||
- A title is a table under `sex`
|
||||
- A table owns the spelling of a selector on it
|
||||
- No builtin reads the clock, so a date is bounded by days, never by an age
|
||||
- `misc` is what every locale shares
|
||||
- A register's canonical spelling loses to the one its domain writes
|
||||
- `misc.territory` is the spine, and a `misc` table naming a territory links to it
|
||||
- A table whose register publishes no frequency draws evenly
|
||||
- `misc.timezone` weighs a zone by the people living in it
|
||||
- `misc.car` is one flat table, not a make linked to its models
|
||||
- `misc.territory` names its sovereign in a column, and there is no `misc.country` table
|
||||
- `misc.loglevel` is a table keyed by POSIX's keyword, carrying the code
|
||||
- `misc.tld` keys carry the leading dot, where other tables key on a bare code
|
||||
- `misc.tld` is a table of its own, and `misc.territory.tld` stays a column
|
||||
- `misc.territory` carries a currency code, it does not link to `misc.currency`
|
||||
- An extension may name two media types
|
||||
- The Swedish ids draw Skatteverket's test series
|
||||
- The US given names come from a mirror of the SSA file
|
||||
- `List` advertises direct descents only
|
||||
- A path draws through its compiled steps
|
||||
- A name lives in the category binding it, or in the repeat or choice item binding it, and is drawn on its first read
|
||||
- comprehension floor: every dimension and the overall at 7.0 or above, from 0.2.0
|
||||
- The template engine stays the root package until item 107; what reads no engine type, a table's rows included, sits in `internal/`
|
||||
- Goal 5.1's `{…}` covers `{{`, `}}` and a lone `}`, so the goal names no escape
|
||||
- Goal 7.2 holds while a first template can be written with no escape; a literal brace still takes `{{` or `}}`
|
||||
- The locales are Ethnologue 2026's ten most-spoken languages, each in the country with the most speakers of it, the Nordic countries and Germany
|
||||
- A country with no open address register covering it whole ships its places and postal codes without streets
|
||||
- The apt, dnf and apk repositories are static files on GitHub Pages, holding the latest release only
|
||||
- The image is pushed to GHCR and copied to Docker Hub, whose name the README shows
|
||||
- A known vulnerability in the Go version or a required module blocks a release, even in code fejkdata never calls, and blocks no merge
|
||||
|
||||
# Scoring run
|
||||
|
||||
The project values for `~/.claude/skills/comprehension-panel/scoring-run.md`:
|
||||
|
||||
- `{language}`: Go
|
||||
- `{kind}`: a Go library and CLI that renders fake data from JSON templates
|
||||
- `{domain}`: fake test-data generation and template languages
|
||||
- `{domain docs}`: the documentation of any fake-data library, such as Faker
|
||||
- `{3am question}`: a seeded render of `sv_SE.address` prints the postal code of one postort beside a street of another
|
||||
- **Shipped-data de-duplication (2026-09-02).** `email.json`'s `local` is a
|
||||
drifted copy of `username.json`, and no shipped file yet uses a held path, an
|
||||
operand or a reference. Fixed in the data fill before the first tag, when the
|
||||
shipped set is rewritten anyway; premise: nothing depends on the shipped data's
|
||||
shape until then. Not raised in review before that.
|
||||
|
||||
-106
@@ -1,106 +0,0 @@
|
||||
# Changelog
|
||||
|
||||
Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). A release's
|
||||
`Breaking` section comes first and names each rejected spelling with its
|
||||
replacement, and each removed path, column or flag.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- Prebuilt binaries for Linux, macOS and Windows on amd64 and arm64, packages for apt,
|
||||
dnf, apk, Homebrew and Scoop, and the image `larvit/fejkdata` on Docker Hub, also at `ghcr.io/larvit/fejkdata`, so
|
||||
the CLI installs without Go. 0.1.0 ships them too; the README's
|
||||
[Install](https://github.com/larvit/fejkdata#install) lists each command.
|
||||
|
||||
## [0.1.0] - 2026-10-13
|
||||
|
||||
The first release: a Go library and a CLI that render locale-aware fake data from JSON
|
||||
templates, and the data they ship with. The
|
||||
[README](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md) documents each part.
|
||||
Install the CLI with `go install github.com/larvit/fejkdata/cmd/fejkdata@v0.1.0`, or the
|
||||
library with `go get github.com/larvit/fejkdata@v0.1.0`. Before 1.0.0 a minor may break,
|
||||
and any release may change what a seed draws, so pin one exact version
|
||||
([Versioning](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#versioning)).
|
||||
|
||||
### Added
|
||||
|
||||
- The CLI, `fejkdata`, renders exactly the template on its stdin and adds nothing:
|
||||
`echo '{/sv_SE.person} hihi' | fejkdata` prints e.g. `Sara Eriksson hihi`. `--seed`
|
||||
repeats a run's output while the version and data stay the same, and without it each
|
||||
run takes a new seed; `-n` renders the template that many times, and `--list`
|
||||
prints the paths the data offers.
|
||||
`--data-path` layers a directory of your own data over the shipped set, and
|
||||
`--no-shipped-data` loads that directory alone. `--format json|ndjson|csv|sql` writes
|
||||
a record's columns, one record per row. The CLI exits `0` on success, `1` on a runtime
|
||||
error such as data that fails to load, and `2` on misuse.
|
||||
- The library, `github.com/larvit/fejkdata`, for Go 1.22 or later on a 64-bit target.
|
||||
`New` loads the modules `WithDataFS` names and the folders `WithDataPath` names, drawing
|
||||
from the seed `WithSeed` gives. Without a seed, `New` fails and names `WithSeed`;
|
||||
without data, it fails and names `WithDataFS`. The same seed repeats the draws
|
||||
while the version and data stay the same. `Fake` renders a path, and `FakeTemplate`
|
||||
and `NewTemplate` render a template. `FakeRecord`, `FakeRecordTemplate` and
|
||||
`NewRecordTemplate` render a record, whose `JSON`, `CSVHeader`, `CSVLine` and
|
||||
`SQLInsert` methods write its columns as the CLI does. `FakeStruct` fills a struct's
|
||||
`fake:"…"` tagged fields. `List` gives the paths `Fake` renders, direct descents only,
|
||||
and `IsTemplate` tells an inline template from a path. Data that fails to load gives
|
||||
an error matching `ErrLoad`, and a record with no fields gives one matching
|
||||
`ErrNoColumns`.
|
||||
- Package `github.com/larvit/fejkdata/cli` is the `fejkdata` command:
|
||||
`cli.Main(modules...)` runs it over the modules given, so
|
||||
`cli.Main(append(data.Modules(), mydata)...)` builds a CLI carrying your own data.
|
||||
That CLI names itself by the file it runs from, and its `--version` prints its own
|
||||
module's version before fejkdata's.
|
||||
- The template language
|
||||
([Data format](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data-format)).
|
||||
Every value in a data file is a string, a choice, a template, a table, a number or a
|
||||
boolean. A `{…}` token reads a sibling field, a name, or a reference from the data
|
||||
root, the file's folder or the folder above (`{/sv_SE.person.last}`, `{.x}`, `{..x}`),
|
||||
or it calls a builtin. Every `{…}` draws afresh, and a name is the one way to keep a
|
||||
pick: `{/sv_SE.person as p}{p.first} {p.last}` is one person. The builtins draw check
|
||||
digits (`luhn`, `mod11`, `ean`), samples (`digits`, `int`, `float`, `uuid`, `ulid`,
|
||||
`iban`, `date`, `time` and more), a counter (`seq`), arithmetic (`calc`) and
|
||||
transforms (`lowercase`, `uppercase`, `ascii`). `New` refuses data that breaks the
|
||||
grammar, could mean two things or cannot render a valid value.
|
||||
- Records ([Records](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#records)):
|
||||
a template seen as columns. A column is a string unless it declares a `datatype`,
|
||||
`integer`, `number` or `boolean`, and a `null` item draws it as null.
|
||||
- Tables ([Table](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#table)): a
|
||||
category naming a `rows` TSV beside it, with the options `key`, `name`, `weight` and
|
||||
`parent`. A path selects a row by key or name, `misc.territory[SE]`, descends to a
|
||||
linked table, `geo.SE.locality[Lund].street`, and steps up to the parent row,
|
||||
`{/misc.timezone[Europe/Oslo]..territory.name}`.
|
||||
- Data sources ([Data](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data)):
|
||||
a directory is a namespace, and each JSON file in it a category. A `.fejkdata.json`
|
||||
manifest at a source's root may carry a `namespace`, the dot path the source mounts
|
||||
under, and an `index`, with which each category loads on the first call reaching it.
|
||||
- The shipped data, in Go packages that each export `FS` and `Modules()`:
|
||||
`github.com/larvit/fejkdata/data/en_US` (package `enus`), `…/sv_SE` (`svse`),
|
||||
`…/misc`, `…/geo/SE` (`geose`) and `…/geo/US` (`geous`). `data.Modules()`, from
|
||||
`github.com/larvit/fejkdata/data`, lists every one, and the CLI carries them all.
|
||||
[`DATA-LICENSES.md`](https://github.com/larvit/fejkdata/blob/v0.1.0/DATA-LICENSES.md)
|
||||
names each source.
|
||||
- `sv_SE` and `en_US`: an `address`, a `person` and more per locale. `sex`,
|
||||
`first-name` and `last-name` are tables weighted by bearers, from SCB, the SSA and
|
||||
the Census Bureau. A person's first name and sex agree, and so does an `en_US`
|
||||
person's title; `person.prefix` is null where a person has no title.
|
||||
`sv_SE.personnummer` and `sv_SE.samordningsnummer` draw Skatteverket's test series.
|
||||
`en_US.ssn` and `en_US.itin` draw the ranges the SSA and the IRS assign, with the
|
||||
columns `area`, `group` and `serial`, and `en_US.phone` draws an exchange NANPA may
|
||||
assign.
|
||||
- `misc`: IANA's registries of HTTP methods, protocols, ports and root-zone TLDs, the
|
||||
HTTP status codes, ISO 3166-1 territories, ISO 4217 currencies with their CLDR
|
||||
symbols, ISO 639 languages, the syslog severities, mime-db's media types, tzdb's
|
||||
zones weighted by the people living in each, user agents and cars, all as tables;
|
||||
and coordinates, credit card numbers, RFC 3339 instants, emoji, MAC addresses,
|
||||
ObjectIds and UUIDs.
|
||||
- `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`,
|
||||
`locality`, `postal-code` and `street`, weighted by population and address counts
|
||||
and built from SCB, GeoNames, Trafikverket NVDB, the US Census Bureau and Wikidata,
|
||||
and an `address` record that draws a street and a postal code inside one
|
||||
locality; `sv_SE.address` and `en_US.address` print it. Every Swedish postort
|
||||
(postal town) the import can place ships with all its streets. A US place ships
|
||||
only if it is incorporated and holds most of some postal code's land. Of those
|
||||
places, every state capital but Honolulu ships, as does every place of 100,000
|
||||
people or more and a tenth of all of them drawn by population. Each US place
|
||||
carries a quarter of its streets, at least ten.
|
||||
@@ -1,31 +0,0 @@
|
||||
# Data licenses
|
||||
|
||||
Every shipped dataset, its source, its licence and the attribution it asks for. A
|
||||
`data-import/` script rebuilds each sourced table, run as the README's
|
||||
[Development](README.md#development) section says; a curated one is hand-written.
|
||||
|
||||
| Table | Source | Licence | Attribution | Rebuild |
|
||||
|-------|--------|---------|-------------|---------|
|
||||
| `geo/SE/locality.tsv`, `postal-code.tsv` | [GeoNames](https://www.geonames.org/) postal codes for SE; populations from SCB tätorter 2023 | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/); CC0 1.0 | "Postal codes from GeoNames, www.geonames.org" | `data-import/geo-se.py` |
|
||||
| `geo/SE/region.tsv`, `municipality.tsv` | [SCB](https://www.scb.se/) län and kommun codes 2026 and population 2024 | [CC0 1.0](https://creativecommons.org/publicdomain/zero/1.0/) | none required | `data-import/geo-se.py` |
|
||||
| `geo/SE/street.tsv` | [Trafikverket NVDB](https://www.trafikverket.se/) Gatunamn, through the open API | CC0 1.0 | none required | `data-import/geo-se.py` |
|
||||
| `geo/US/region.tsv`, `municipality.tsv`, `locality.tsv` | [Census Bureau](https://www.census.gov/) Gazetteer 2026 and population estimates 2025; which places are state capitals from [Wikidata](https://www.wikidata.org/) | [public domain](https://www.usa.gov/government-works); CC0 1.0 | none required | `data-import/geo-us.py` |
|
||||
| `geo/US/postal-code.tsv`, `street.tsv` | Census Bureau ZCTA to place relationships 2020 and TIGER/Line 2025 address ranges and feature names | public domain | none required | `data-import/geo-us.py` |
|
||||
| `sv_SE/first-name.tsv`, `last-name.tsv` | [SCB](https://www.scb.se/) names with at least two bearers, 31 December 2022 | CC0 1.0 | "Källa: SCB" | `data-import/names-se.py` |
|
||||
| `en_US/first-name.tsv` | [SSA](https://www.ssa.gov/oact/babynames/) baby names, births 1930 to 2020, through [hackerb9/ssa-baby-names](https://github.com/hackerb9/ssa-baby-names) | public domain | none required | `data-import/names-us.py` |
|
||||
| `en_US/last-name.tsv` | Census Bureau surnames occurring 100 or more times, 2010 | public domain | none required | `data-import/names-us.py` |
|
||||
| `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv`, `misc/car.tsv` | curated; Skatteverket's test birth numbers are facts, and `car.tsv` waits for an international source (goal 13) | — | — | — |
|
||||
| `misc/currency.tsv` | [datasets/currency-codes](https://github.com/datasets/currency-codes); symbols from [Unicode CLDR](https://github.com/unicode-org/cldr) `en.xml` and `root.xml` | PDDL 1.0; [Unicode License v3](https://www.unicode.org/license.txt) | CLDR: "Copyright © 1991-2025 Unicode, Inc. Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries." | `data-import/currency.py` |
|
||||
| `misc/httpmethod.tsv` | [IANA HTTP Method Registry](https://www.iana.org/assignments/http-methods/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpmethod.py` |
|
||||
| `misc/httpstatus.tsv` | [IANA HTTP Status Code Registry](https://www.iana.org/assignments/http-status-codes/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpstatus.py` |
|
||||
| `misc/language.tsv` | [datasets/language-codes](https://github.com/datasets/language-codes), the [Library of Congress](https://www.loc.gov/standards/iso639-2/) ISO 639-2 register | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/language.py` |
|
||||
| `misc/loglevel.tsv` | [RFC 5424](https://www.rfc-editor.org/rfc/rfc5424.txt) Table 2, the syslog message severities; keywords from the [POSIX](https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/syslog.h.html) `<syslog.h>` severity macros | facts | none required | `data-import/loglevel.py` |
|
||||
| `misc/mimetype.tsv` | [mime-db](https://github.com/jshttp/mime-db), the IANA media type registry with filename extensions | [MIT](https://github.com/jshttp/mime-db/blob/master/LICENSE) | "Copyright (c) 2014 Jonathan Ong, Copyright (c) 2015-2022 Douglas Christopher Wilson" | `data-import/mimetype.py` |
|
||||
| `misc/port.tsv` | [IANA Service Name and Transport Protocol Port Number Registry](https://www.iana.org/assignments/service-names-port-numbers/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/port.py` |
|
||||
| `misc/protocol.tsv` | [IANA Protocol Numbers](https://www.iana.org/assignments/protocol-numbers/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/protocol.py` |
|
||||
| `misc/territory.tsv` | [datasets/country-codes](https://github.com/datasets/country-codes) | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/territory.py` |
|
||||
| `misc/timezone.tsv` | [IANA tzdb](https://www.iana.org/time-zones) 2026d `zone.tab` and the standard offset of each zone; weights from [GeoNames](https://www.geonames.org/) `cities15000` populations | [public domain](https://data.iana.org/time-zones/tzdb/LICENSE); [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) | "Populations from GeoNames, www.geonames.org" | `data-import/timezone.py` |
|
||||
| `misc/tld.tsv` | [IANA Root Zone Database](https://www.iana.org/domains/root/db), checked against the root zone's [TLD list](https://data.iana.org/TLD/tlds-alpha-by-domain.txt) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/tld.py` |
|
||||
| `misc/useragent.tsv` | [top-user-agents](https://github.com/microlinkhq/top-user-agents) desktop and mobile lists | [MIT](https://github.com/microlinkhq/top-user-agents/blob/master/LICENSE.md) | "Copyright © 2020 Kiko Beats" | `data-import/useragent.py` |
|
||||
|
||||
Every other category is hand-written under [`data/`](data), MIT like the code.
|
||||
+9
-33
@@ -1,43 +1,19 @@
|
||||
ARG GO_VERSION=1.27.2
|
||||
FROM golang:${GO_VERSION} AS portable
|
||||
# The merge gate: `docker build .` fails if vet, formatting or tests fail, and CI
|
||||
# runs exactly this. Day-to-day, prefer `docker compose run` (bind-mounts source,
|
||||
# no rebuilds). GO_VERSION defaults to the latest stable Go; override it to test
|
||||
# the lowest supported version: docker build --build-arg GO_VERSION=1.22.12 .
|
||||
ARG GO_VERSION=1.26.4
|
||||
FROM golang:${GO_VERSION}
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Module layer cached separately from source.
|
||||
COPY go.mod ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
RUN go vet ./... && \
|
||||
go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test . && \
|
||||
{ unformatted="$(gofmt -l .)"; test -z "$unformatted" || \
|
||||
{ echo "unformatted files:"; echo "$unformatted"; exit 1; }; } && \
|
||||
go test -race ./...
|
||||
|
||||
FROM golang:${GO_VERSION} AS vulnerabilities
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
RUN go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan module -format json > /report.json
|
||||
|
||||
FROM scratch AS vulnerability-report
|
||||
COPY --from=vulnerabilities /report.json /
|
||||
|
||||
FROM python:3.14.8-slim AS release-tooling
|
||||
COPY release-tooling/ /release-tooling/
|
||||
RUN python3 -m unittest discover -s /release-tooling -p '*_test.py' && touch /tested
|
||||
|
||||
FROM python:3.14.8-slim AS vulnerability-filing
|
||||
COPY vulnerability-filing/ /vulnerability-filing/
|
||||
# The filer's test files an item into the plan itself.
|
||||
COPY todo.md /todo.md
|
||||
RUN python3 -m unittest discover -s /vulnerability-filing -p '*_test.py' && touch /tested
|
||||
|
||||
FROM goreleaser/goreleaser:v2.18.3 AS packaging
|
||||
WORKDIR /check
|
||||
COPY release-tooling/goreleaser.yaml ./
|
||||
RUN git init -q && git remote add origin https://github.com/larvit/fejkdata.git && goreleaser check --config goreleaser.yaml && touch /tested
|
||||
|
||||
FROM portable
|
||||
# BuildKit skips a stage nothing copies from.
|
||||
COPY --from=release-tooling /tested /tmp/release-tooling-tested
|
||||
COPY --from=vulnerability-filing /tested /tmp/vulnerability-filing-tested
|
||||
COPY --from=packaging /tested /tmp/packaging-tested
|
||||
RUN unformatted="$(gofmt -l .)"; test -z "$unformatted" || \
|
||||
{ echo "unformatted files:"; echo "$unformatted"; exit 1; }
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Timewave
|
||||
Copyright (c) 2026 Larvit
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
package fejkdata
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/larvit/fejkdata/internal/builtinfunc"
|
||||
"github.com/larvit/fejkdata/internal/grammar"
|
||||
"github.com/larvit/fejkdata/internal/invariant"
|
||||
)
|
||||
|
||||
// arm is one alternative of a {a|b} token or one operand, split into the head,
|
||||
// whose node `template.startOf` finds, and the tail of a dotted path into it.
|
||||
type arm struct {
|
||||
spelling string // as written, for messages
|
||||
head string
|
||||
tail []string
|
||||
levels []pickKey // the key of each level the steps pass, from the head they start at to the leaf; a step's at indexes it
|
||||
steps []pathStep
|
||||
leaves []node // every node the path may land on, one per variant it passes
|
||||
kind armKind
|
||||
named *nameBinding // namedRead: the binding of the name it reads through
|
||||
}
|
||||
|
||||
// armKind is how expand reads an arm, fixed at compile.
|
||||
type armKind uint8
|
||||
|
||||
const (
|
||||
freshRead armKind = iota // drawn afresh at every read
|
||||
namedRead // read through a name, kept in its pick
|
||||
)
|
||||
|
||||
// splitArm splits one name into head and tail. refs maps a reference to what
|
||||
// resolveRefs resolved it to; before that, a reference is whole.
|
||||
func splitArm(name string, refs map[string]resolvedRef) arm {
|
||||
if grammar.IsRef(name) {
|
||||
if r, resolved := refs[name]; resolved {
|
||||
return pathArm(name, r.head, r.tail)
|
||||
}
|
||||
return pathArm(name, name, nil)
|
||||
}
|
||||
segs, err := grammar.SplitPath(name)
|
||||
if err != nil || len(segs) == 1 {
|
||||
return pathArm(name, name, nil)
|
||||
}
|
||||
return pathArm(name, segs[0], segs[1:])
|
||||
}
|
||||
|
||||
func pathArm(name, head string, segs []string) arm {
|
||||
return arm{spelling: name, head: head, tail: segs, levels: levelKeys(append([]string{head}, segs...), 1)}
|
||||
}
|
||||
|
||||
// key is the key of the leaf a lands on, the one key every way of writing this read shares.
|
||||
func (a arm) key() pickKey { return a.levels[len(a.levels)-1] }
|
||||
|
||||
func checkSegments(a arm) error {
|
||||
if len(a.tail) == 0 {
|
||||
return nil
|
||||
}
|
||||
return grammar.CheckSegments(append([]string{a.head}, a.tail...))
|
||||
}
|
||||
|
||||
// op is one compiled unit of a format string: a literal run, a name read,
|
||||
// or a builtin already prepared with its args.
|
||||
type op struct {
|
||||
grammar.Token
|
||||
arms []arm // grammar.PathRead: the '|' alternatives, split into head and tail once
|
||||
call builtinfunc.Call
|
||||
// operands are the fields the builtin reads, in the order its operands func
|
||||
// fixed; expand reads them before the call. nil for a builtin that reads none.
|
||||
operands []arm
|
||||
}
|
||||
|
||||
func (t *template) compileArms(names []string, targets map[*nameBinding]nameTarget) ([]arm, error) {
|
||||
if len(names) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
arms := make([]arm, len(names))
|
||||
for i, name := range names {
|
||||
a, err := t.compileArm(name, targets)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
arms[i] = a
|
||||
}
|
||||
return arms, nil
|
||||
}
|
||||
|
||||
// compileArm compiles one read into a path: from the head it names, or, for a read through a
|
||||
// name, from what that name binds.
|
||||
func (t *template) compileArm(name string, targets map[*nameBinding]nameTarget) (arm, error) {
|
||||
a := splitArm(name, t.refs.byName)
|
||||
if !t.isName(a.head) {
|
||||
start := t.startOf(a.head)
|
||||
if start == nil {
|
||||
panic(invariant.Broken("{%s} reads a head nothing resolved", a.spelling))
|
||||
}
|
||||
w := compilePath(start, a.tail)
|
||||
a.steps, a.leaves = w.steps, w.leaves
|
||||
return a, nil
|
||||
}
|
||||
b := t.nameScope.lookup(a.head)
|
||||
if grammar.HasSelector(a.tail) {
|
||||
if b.bindsField() {
|
||||
return a, fmt.Errorf("a path through name %q may not select a row; bind the row to a name of its own", a.head)
|
||||
}
|
||||
return a, fmt.Errorf("a path through name %q may not select a row; read the row without the name, {%s}, or bind it to a name of its own", a.head, grammar.JoinSegments(append([]string{b.ref}, a.tail...)))
|
||||
}
|
||||
target := targets[b]
|
||||
full := append(target.tail[:len(target.tail):len(target.tail)], a.tail...)
|
||||
if err := provePath(target.start, full, a.head, len(target.tail)); err != nil {
|
||||
return a, err
|
||||
}
|
||||
w := compilePath(target.start, full)
|
||||
a.kind, a.named, a.steps, a.leaves = namedRead, b, w.steps, w.leaves
|
||||
a.levels = levelKeys(full, 0)
|
||||
return a, nil
|
||||
}
|
||||
+32
-44
@@ -6,17 +6,17 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func benchGenerator(b *testing.B, data Option) *Generator {
|
||||
func benchGenerator(b *testing.B, dir string) *Generator {
|
||||
b.Helper()
|
||||
f, err := New(data, WithSeed(1))
|
||||
f, err := New(WithoutShippedData(), WithDataPath(dir), WithSeed(1))
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func benchPath(b *testing.B, data Option, path string) {
|
||||
f := benchGenerator(b, data)
|
||||
func benchPath(b *testing.B, dir, path string) {
|
||||
f := benchGenerator(b, dir)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
@@ -26,12 +26,12 @@ func benchPath(b *testing.B, data Option, path string) {
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkPerson(b *testing.B) { benchPath(b, withShipped(), "sv_SE.person") }
|
||||
func BenchmarkAddress(b *testing.B) { benchPath(b, withShipped(), "sv_SE.address") }
|
||||
func BenchmarkWord(b *testing.B) { benchPath(b, withShipped(), "sv_SE.word") }
|
||||
func BenchmarkCreditcard(b *testing.B) { benchPath(b, withShipped(), "misc.creditcard") }
|
||||
func BenchmarkPersonnummer(b *testing.B) { benchPath(b, withShipped(), "sv_SE.personnummer") }
|
||||
func BenchmarkUUIDv7(b *testing.B) { benchPath(b, withShipped(), "misc.uuid") }
|
||||
func BenchmarkPerson(b *testing.B) { benchPath(b, "data/sv_SE", "person") }
|
||||
func BenchmarkAddress(b *testing.B) { benchPath(b, "data/sv_SE", "address") }
|
||||
func BenchmarkWord(b *testing.B) { benchPath(b, "data/sv_SE", "word") }
|
||||
func BenchmarkCreditcard(b *testing.B) { benchPath(b, "data/misc", "creditcard") }
|
||||
func BenchmarkSSN(b *testing.B) { benchPath(b, "data/sv_SE", "ssn") }
|
||||
func BenchmarkUUIDv7(b *testing.B) { benchPath(b, "data/misc", "uuid") }
|
||||
|
||||
func tmpData(b *testing.B, name, body string) string {
|
||||
b.Helper()
|
||||
@@ -44,64 +44,52 @@ func tmpData(b *testing.B, name, body string) string {
|
||||
|
||||
func BenchmarkCalc(b *testing.B) {
|
||||
dir := tmpData(b, "inv", `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`)
|
||||
benchPath(b, WithDataPath(dir), "inv")
|
||||
benchPath(b, dir, "inv")
|
||||
}
|
||||
|
||||
func BenchmarkLongLiteral(b *testing.B) {
|
||||
dir := tmpData(b, "sql", `{"format":"INSERT INTO customers (id, name, city) VALUES (123, '{word}', '{word}');","word":["alpha","beta","gamma","delta"]}`)
|
||||
benchPath(b, WithDataPath(dir), "sql")
|
||||
benchPath(b, dir, "sql")
|
||||
}
|
||||
|
||||
func BenchmarkRepeat(b *testing.B) {
|
||||
dir := tmpData(b, "many", `{"format":"{word}","repeat":20,"separator":", ","word":["alpha","beta","gamma","delta"]}`)
|
||||
benchPath(b, WithDataPath(dir), "many")
|
||||
benchPath(b, dir, "many")
|
||||
}
|
||||
|
||||
// BenchmarkNamed measures a format that binds a field to a name and reads two paths through
|
||||
// it — what a correlated pair costs against BenchmarkUnnamed, the same output drawn from two
|
||||
// independent fields.
|
||||
func BenchmarkNamed(b *testing.B) {
|
||||
dir := tmpData(b, "addr", `{"format":"{place as p}{p.postal-code} {p.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`)
|
||||
benchPath(b, WithDataPath(dir), "addr")
|
||||
// BenchmarkBound measures a format that binds a head and reads two paths from it —
|
||||
// what a correlated pair costs against BenchmarkUnbound, the same output drawn from
|
||||
// two independent fields.
|
||||
func BenchmarkBound(b *testing.B) {
|
||||
dir := tmpData(b, "addr", `{"format":"{place.postal-code} {place.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`)
|
||||
benchPath(b, dir, "addr")
|
||||
}
|
||||
|
||||
func BenchmarkUnnamed(b *testing.B) {
|
||||
func BenchmarkUnbound(b *testing.B) {
|
||||
dir := tmpData(b, "addr", `{"format":"{postal-code} {locality}","postal-code":["1{digits(2)} {digits(2)}","573 {digits(2)}"],"locality":["Stockholm","Tranås"]}`)
|
||||
benchPath(b, WithDataPath(dir), "addr")
|
||||
benchPath(b, dir, "addr")
|
||||
}
|
||||
|
||||
// BenchmarkNamedDeep reads two paths through a name and an intermediate level.
|
||||
func BenchmarkNamedDeep(b *testing.B) {
|
||||
dir := tmpData(b, "addr", `{"format":"{p as q}{q.addr.city} {q.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`)
|
||||
benchPath(b, WithDataPath(dir), "addr")
|
||||
// BenchmarkBoundDeep reads two paths through an intermediate level, the shape the
|
||||
// depth question is about.
|
||||
func BenchmarkBoundDeep(b *testing.B) {
|
||||
dir := tmpData(b, "addr", `{"format":"{p.addr.city} {p.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`)
|
||||
benchPath(b, dir, "addr")
|
||||
}
|
||||
|
||||
// BenchmarkNamedWide reads ten paths through one name: the point where the maps a pick keeps
|
||||
// outgrow a single bucket.
|
||||
func BenchmarkNamedWide(b *testing.B) {
|
||||
dir := tmpData(b, "row", `{"format":"{r as s}{s.a}{s.b}{s.c}{s.d}{s.e}{s.f}{s.g}{s.h}{s.i}{s.j}","r":[
|
||||
// BenchmarkBoundWide reads ten paths from one row: the point where the maps an
|
||||
// expansion holds outgrow a single bucket.
|
||||
func BenchmarkBoundWide(b *testing.B) {
|
||||
dir := tmpData(b, "row", `{"format":"{r.a}{r.b}{r.c}{r.d}{r.e}{r.f}{r.g}{r.h}{r.i}{r.j}","r":[
|
||||
{"format":"x","a":"1","b":"2","c":"3","d":"4","e":"5","f":"6","g":"7","h":"8","i":"9","j":"0"},
|
||||
{"format":"y","a":"A","b":"B","c":"C","d":"D","e":"E","f":"F","g":"G","h":"H","i":"I","j":"J"}]}`)
|
||||
benchPath(b, WithDataPath(dir), "row")
|
||||
}
|
||||
|
||||
func BenchmarkFirstFake(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
f, err := New(withShipped(), WithSeed(1))
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if _, err := f.Fake("sv_SE.address"); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
benchPath(b, dir, "row")
|
||||
}
|
||||
|
||||
func BenchmarkNew(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
if _, err := New(withShipped(), WithSeed(1)); err != nil {
|
||||
if _, err := New(WithSeed(1)); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
+414
@@ -0,0 +1,414 @@
|
||||
package fejkdata
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// maxLen caps sample output lengths (hex, nanoid, base64) and the renders a repeat
|
||||
// multiplies to along any path; maxDecimals caps float/calc decimal places. So a
|
||||
// fat-fingered or overflowing argument fails at New instead of trying to allocate
|
||||
// gigabytes — or panicking — at render.
|
||||
const (
|
||||
maxLen = 1 << 20 // 1,048,576 chars/bytes
|
||||
maxDecimals = 1024
|
||||
)
|
||||
|
||||
// builtins is the registry of {name(args)} functions. Derivations read the digits
|
||||
// emitted so far in the current expansion (place them after their payload);
|
||||
// samples read only the rng. A time-based id (uuid v7, ulid) draws its timestamp
|
||||
// from the rng, not the wall clock, so seeded output stays reproducible.
|
||||
var builtins = map[string]builtin{
|
||||
"luhn": {arity: 0, prep: derive(func(e string) string { return string(rune('0' + luhnCheck(e))) })},
|
||||
"mod11": {arity: 0, prep: derive(mod11Check)},
|
||||
"ean": {arity: 0, prep: derive(eanCheck)},
|
||||
"uuid": {arity: 0, prep: sample(uuidV7)},
|
||||
"ulid": {arity: 0, prep: sample(ulid)},
|
||||
"nanoid": {arity: 1, check: posIntArg, prep: chars(nanoidAlphabet)},
|
||||
"hex": {arity: 1, check: posIntArg, prep: chars(hexDigits)},
|
||||
"digits": {arity: 1, check: posIntArg, prep: chars("0123456789")},
|
||||
"upper": {arity: 1, check: posIntArg, prep: chars("ABCDEFGHIJKLMNOPQRSTUVWXYZ")},
|
||||
"lower": {arity: 1, check: posIntArg, prep: chars("abcdefghijklmnopqrstuvwxyz")},
|
||||
"base64": {arity: 1, check: posIntArg, prep: func(a []string) callFn {
|
||||
n := atoi(a[0])
|
||||
return func(s *session, _ string, _ []string) string {
|
||||
return base64.StdEncoding.EncodeToString(randBytes(s, n))
|
||||
}
|
||||
}},
|
||||
"int": {arity: 2, check: intRangeArgs, prep: func(a []string) callFn {
|
||||
lo, span := atoi(a[0]), atoi(a[1])-atoi(a[0])+1
|
||||
return func(s *session, _ string, _ []string) string { return strconv.Itoa(lo + s.IntN(span)) }
|
||||
}},
|
||||
"float": {arity: 3, check: floatArgs, prep: func(a []string) callFn {
|
||||
lo, hi, dp := atof(a[0]), atof(a[1]), atoi(a[2])
|
||||
return func(s *session, _ string, _ []string) string {
|
||||
return strconv.FormatFloat(lo+s.Float64()*(hi-lo), 'f', dp, 64)
|
||||
}
|
||||
}},
|
||||
"iban": {arity: 1, check: ibanArg, prep: func(a []string) callFn {
|
||||
cc := a[0]
|
||||
return func(s *session, _ string, _ []string) string { return iban(s, cc) }
|
||||
}},
|
||||
"calc": {arity: -1, check: checkCalc, prep: calcPrep, operands: calcOperands},
|
||||
"lowercase": {arity: 1, check: transformArg, prep: transformPrep(strings.ToLower), operands: transformOperand},
|
||||
"uppercase": {arity: 1, check: transformArg, prep: transformPrep(strings.ToUpper), operands: transformOperand},
|
||||
"ascii": {arity: 1, check: transformArg, prep: transformPrep(asciiFold), operands: transformOperand},
|
||||
// seq is the one stateful builtin: a per-session counter from 1, advancing on
|
||||
// each call. An optional name selects an independent counter; no name uses the
|
||||
// default one. Deterministic by construction, so seeded output stays stable.
|
||||
"seq": {arity: -1, check: seqArg, prep: func(a []string) callFn {
|
||||
key := ""
|
||||
if len(a) == 1 {
|
||||
key = a[0]
|
||||
}
|
||||
return func(s *session, _ string, _ []string) string {
|
||||
return strconv.FormatUint(s.next(key), 10)
|
||||
}
|
||||
}},
|
||||
}
|
||||
|
||||
// derive and sample are the two argument-free builtin shapes: a derivation reads
|
||||
// the output emitted so far, a sample reads only the rng. chars is the shape of a
|
||||
// sample of n characters drawn from an alphabet.
|
||||
func derive(f func(emitted string) string) func([]string) callFn {
|
||||
return func([]string) callFn {
|
||||
return func(_ *session, emitted string, _ []string) string { return f(emitted) }
|
||||
}
|
||||
}
|
||||
|
||||
func sample(f func(rng) string) func([]string) callFn {
|
||||
return func([]string) callFn {
|
||||
return func(s *session, _ string, _ []string) string { return f(s) }
|
||||
}
|
||||
}
|
||||
|
||||
func chars(alphabet string) func([]string) callFn {
|
||||
return func(a []string) callFn {
|
||||
n := atoi(a[0])
|
||||
return func(s *session, _ string, _ []string) string { return randChars(s, n, alphabet) }
|
||||
}
|
||||
}
|
||||
|
||||
const hexDigits = "0123456789abcdef"
|
||||
|
||||
// transforms are the builtins that rewrite one operand's value; they nest, so
|
||||
// {lowercase(ascii(x))} folds then lowers.
|
||||
var transforms = map[string]func(string) string{
|
||||
"ascii": asciiFold,
|
||||
"lowercase": strings.ToLower,
|
||||
"uppercase": strings.ToUpper,
|
||||
}
|
||||
|
||||
// unwrapTransform peels nested transform calls off an operand arg, returning the
|
||||
// field it finally names and the transforms to apply, innermost last.
|
||||
func unwrapTransform(arg string) (leaf string, chain []func(string) string, err error) {
|
||||
for {
|
||||
name, args, isCall := funcCall(arg)
|
||||
if !isCall {
|
||||
return arg, chain, nil
|
||||
}
|
||||
fn, isTransform := transforms[name]
|
||||
if !isTransform {
|
||||
return "", nil, fmt.Errorf("%s(%s) is not a transform, so it cannot be an operand", name, strings.Join(args, ","))
|
||||
}
|
||||
if len(args) != 1 {
|
||||
return "", nil, fmt.Errorf("%s takes 1 arg, got %d", name, len(args))
|
||||
}
|
||||
chain = append(chain, fn)
|
||||
arg = args[0]
|
||||
}
|
||||
}
|
||||
|
||||
func transformArg(fields map[string]node, a []string) error {
|
||||
leaf, _, err := unwrapTransform(a[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isRef(leaf) {
|
||||
_, _, err := refShape(leaf)
|
||||
return err
|
||||
}
|
||||
return checkArm(leaf, fields)
|
||||
}
|
||||
|
||||
func transformOperand(a []string) []string {
|
||||
leaf, _, err := unwrapTransform(a[0])
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return []string{leaf}
|
||||
}
|
||||
|
||||
func transformPrep(outer func(string) string) func([]string) callFn {
|
||||
return func(a []string) callFn {
|
||||
_, chain, err := unwrapTransform(a[0])
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("fejkdata: transform arg %q reached prep unvalidated: %v", a[0], err))
|
||||
}
|
||||
return func(_ *session, _ string, operands []string) string {
|
||||
v := operands[0]
|
||||
for i := len(chain) - 1; i >= 0; i-- {
|
||||
v = chain[i](v)
|
||||
}
|
||||
return outer(v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// asciiFolds maps the Latin letters with diacritics or ligatures to ASCII.
|
||||
var asciiFolds = map[rune]string{
|
||||
'À': "A", 'Á': "A", 'Â': "A", 'Ã': "A", 'Ä': "A", 'Å': "A", 'Æ': "AE", 'Ç': "C",
|
||||
'È': "E", 'É': "E", 'Ê': "E", 'Ë': "E", 'Ì': "I", 'Í': "I", 'Î': "I", 'Ï': "I",
|
||||
'Ð': "D", 'Ñ': "N", 'Ò': "O", 'Ó': "O", 'Ô': "O", 'Õ': "O", 'Ö': "O", 'Ø': "O",
|
||||
'Ù': "U", 'Ú': "U", 'Û': "U", 'Ü': "U", 'Ý': "Y", 'Þ': "Th", 'ß': "ss", 'Œ': "OE",
|
||||
'à': "a", 'á': "a", 'â': "a", 'ã': "a", 'ä': "a", 'å': "a", 'æ': "ae", 'ç': "c",
|
||||
'è': "e", 'é': "e", 'ê': "e", 'ë': "e", 'ì': "i", 'í': "i", 'î': "i", 'ï': "i",
|
||||
'ð': "d", 'ñ': "n", 'ò': "o", 'ó': "o", 'ô': "o", 'õ': "o", 'ö': "o", 'ø': "o",
|
||||
'ù': "u", 'ú': "u", 'û': "u", 'ü': "u", 'ý': "y", 'þ': "th", 'ÿ': "y", 'œ': "oe",
|
||||
}
|
||||
|
||||
// asciiFold rewrites s to ASCII: folded Latin letters stay, any other non-ASCII
|
||||
// rune is dropped.
|
||||
func asciiFold(s string) string {
|
||||
var b strings.Builder
|
||||
for _, r := range s {
|
||||
if r < unicode.MaxASCII {
|
||||
b.WriteRune(r)
|
||||
} else {
|
||||
b.WriteString(asciiFolds[r])
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// atoi parses an arg a builtin's check already validated. It panics rather than
|
||||
// returning zero, so a check that stops covering its own args is a stack trace and
|
||||
// not a silently wrong length, range or decimal count.
|
||||
func atoi(s string) int {
|
||||
n, err := strconv.Atoi(s)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", s, err))
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// atof is atoi for a float arg, and reports an unvalidated one the same way.
|
||||
func atof(s string) float64 {
|
||||
f, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", s, err))
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func randBytes(r rng, n int) []byte {
|
||||
b := make([]byte, n)
|
||||
for i := range b {
|
||||
b[i] = byte(r.IntN(256))
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func randChars(r rng, n int, alphabet string) string {
|
||||
b := make([]byte, n)
|
||||
for i := range b {
|
||||
b[i] = alphabet[r.IntN(len(alphabet))]
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func posIntArg(_ map[string]node, a []string) error {
|
||||
n, err := strconv.Atoi(a[0])
|
||||
if err != nil || n < 1 {
|
||||
return fmt.Errorf("count %q must be a positive integer", a[0])
|
||||
}
|
||||
if n > maxLen {
|
||||
return fmt.Errorf("count %d exceeds the maximum %d", n, maxLen)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func intRangeArgs(_ map[string]node, a []string) error {
|
||||
lo, e1 := strconv.Atoi(a[0])
|
||||
hi, e2 := strconv.Atoi(a[1])
|
||||
if e1 != nil || e2 != nil {
|
||||
return fmt.Errorf("int(min,max) needs integer args, got %q,%q", a[0], a[1])
|
||||
}
|
||||
if lo > hi {
|
||||
return fmt.Errorf("int(min,max): min %d > max %d", lo, hi)
|
||||
}
|
||||
if uint64(hi)-uint64(lo) >= uint64(math.MaxInt64) { // span hi-lo+1 would overflow int -> IntN panic
|
||||
return fmt.Errorf("int(min,max): range %d..%d is too wide", lo, hi)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func floatArgs(_ map[string]node, a []string) error {
|
||||
lo, e1 := strconv.ParseFloat(a[0], 64)
|
||||
hi, e2 := strconv.ParseFloat(a[1], 64)
|
||||
dp, e3 := strconv.Atoi(a[2])
|
||||
if e1 != nil || e2 != nil || e3 != nil {
|
||||
return fmt.Errorf("float(min,max,dp) needs numeric args, got %q,%q,%q", a[0], a[1], a[2])
|
||||
}
|
||||
if lo > hi {
|
||||
return fmt.Errorf("float(min,max,dp): min %v > max %v", lo, hi)
|
||||
}
|
||||
if math.IsInf(hi-lo, 0) { // an overflowing span would render as "+Inf"
|
||||
return fmt.Errorf("float(min,max,dp): range %v..%v is too wide", lo, hi)
|
||||
}
|
||||
if dp < 0 || dp > maxDecimals {
|
||||
return fmt.Errorf("float(min,max,dp): decimals %d out of range 0..%d", dp, maxDecimals)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func seqArg(_ map[string]node, a []string) error {
|
||||
if len(a) > 1 {
|
||||
return fmt.Errorf("seq takes at most one name, got %d args", len(a))
|
||||
}
|
||||
if len(a) == 1 && a[0] == "" {
|
||||
return fmt.Errorf("seq name must not be empty")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// nanoidAlphabet is the 64-char URL-safe set Nano IDs use.
|
||||
const nanoidAlphabet = "_-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
|
||||
// uuidV7 builds an RFC 9562 v7 UUID. The 48-bit timestamp field is drawn from
|
||||
// the rng (not the clock) to stay reproducible, then the version (7) and variant
|
||||
// (10) bits are forced; the rest is random.
|
||||
func uuidV7(r rng) string {
|
||||
b := randBytes(r, 16)
|
||||
b[6] = b[6]&0x0f | 0x70
|
||||
b[8] = b[8]&0x3f | 0x80
|
||||
var sb strings.Builder
|
||||
for i, x := range b {
|
||||
if i == 4 || i == 6 || i == 8 || i == 10 {
|
||||
sb.WriteByte('-')
|
||||
}
|
||||
sb.WriteByte(hexDigits[x>>4])
|
||||
sb.WriteByte(hexDigits[x&0x0f])
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// crockford is the ULID/Crockford base32 alphabet (no I, L, O, U).
|
||||
const crockford = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
|
||||
|
||||
// ulid builds a 26-char ULID: 128 random bits (timestamp from the rng) encoded
|
||||
// big-endian into base32, the 130-bit stream left-padded with two zero bits.
|
||||
func ulid(r rng) string {
|
||||
b := randBytes(r, 16)
|
||||
out := make([]byte, 26)
|
||||
for i := range out {
|
||||
v := 0
|
||||
for j := 0; j < 5; j++ { // 5 bits per char; the two leading pad bits are zero
|
||||
real := 5*i + j - 2
|
||||
bit := 0
|
||||
if real >= 0 {
|
||||
bit = int(b[real/8]>>(7-uint(real%8))) & 1
|
||||
}
|
||||
v = v<<1 | bit
|
||||
}
|
||||
out[i] = crockford[v]
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
// luhnCheck returns the Luhn check digit (0-9) over the digits of s; non-digit
|
||||
// runes are skipped. Doubling runs from the rightmost digit, so the result is
|
||||
// correct whatever the payload length.
|
||||
func luhnCheck(s string) int {
|
||||
sum, double := 0, true
|
||||
for i := len(s) - 1; i >= 0; i-- {
|
||||
c := s[i]
|
||||
if c < '0' || c > '9' {
|
||||
continue
|
||||
}
|
||||
d := int(c - '0')
|
||||
if double {
|
||||
if d *= 2; d > 9 {
|
||||
d -= 9
|
||||
}
|
||||
}
|
||||
double = !double
|
||||
sum += d
|
||||
}
|
||||
return (10 - sum%10) % 10
|
||||
}
|
||||
|
||||
// mod11Check returns the weighted mod-11 check character over the digits of s
|
||||
// (weights 2..7 cycling from the right). A would-be value of 10 emits 'X', as in
|
||||
// ISBN-10 / ISO 7064; non-digits are skipped.
|
||||
func mod11Check(s string) string {
|
||||
sum, w := 0, 2
|
||||
for i := len(s) - 1; i >= 0; i-- {
|
||||
c := s[i]
|
||||
if c < '0' || c > '9' {
|
||||
continue
|
||||
}
|
||||
sum += int(c-'0') * w
|
||||
if w++; w > 7 {
|
||||
w = 2
|
||||
}
|
||||
}
|
||||
if chk := (11 - sum%11) % 11; chk != 10 {
|
||||
return string(rune('0' + chk))
|
||||
}
|
||||
return "X"
|
||||
}
|
||||
|
||||
// eanCheck returns the EAN-13 / UPC-A / ISBN-13 / GTIN check digit over the
|
||||
// digits of s: weights 3 and 1 alternating from the rightmost digit, mod 10.
|
||||
func eanCheck(s string) string {
|
||||
sum, w := 0, 3
|
||||
for i := len(s) - 1; i >= 0; i-- {
|
||||
c := s[i]
|
||||
if c < '0' || c > '9' {
|
||||
continue
|
||||
}
|
||||
sum += int(c-'0') * w
|
||||
w = 4 - w // 3 <-> 1
|
||||
}
|
||||
return string(rune('0' + (10-sum%10)%10))
|
||||
}
|
||||
|
||||
// ibanLen maps a supported country code to the full IBAN length. The check digits
|
||||
// sit between the country code and the BBAN, so — unlike luhn/ean — iban can't be
|
||||
// a left-to-right derivation; it generates the whole value instead.
|
||||
var ibanLen = map[string]int{"BE": 16, "DE": 22, "DK": 18, "ES": 24, "FI": 18, "NO": 15, "SE": 24}
|
||||
|
||||
func ibanArg(_ map[string]node, a []string) error {
|
||||
if _, ok := ibanLen[a[0]]; !ok {
|
||||
return fmt.Errorf("iban(%q): unsupported country code", a[0])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// iban generates a structurally valid IBAN for cc: a numeric BBAN of the right
|
||||
// length, then mod-97 check digits. Real bank/branch structure isn't modelled —
|
||||
// the result passes length and checksum validation, which is what fake data needs.
|
||||
func iban(r rng, cc string) string {
|
||||
bban := make([]byte, ibanLen[cc]-4)
|
||||
for i := range bban {
|
||||
bban[i] = byte('0' + r.IntN(10))
|
||||
}
|
||||
rem := 0
|
||||
feed := func(d int) { rem = (rem*10 + d) % 97 }
|
||||
for _, c := range bban {
|
||||
feed(int(c - '0'))
|
||||
}
|
||||
for i := 0; i < len(cc); i++ { // letters A-Z -> 10..35, fed as two digits
|
||||
v := int(cc[i]-'A') + 10
|
||||
feed(v / 10)
|
||||
feed(v % 10)
|
||||
}
|
||||
feed(0)
|
||||
feed(0)
|
||||
return fmt.Sprintf("%s%02d%s", cc, 98-rem, bban)
|
||||
}
|
||||
+38
-224
@@ -2,14 +2,9 @@ package fejkdata
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/larvit/fejkdata/internal/invariant"
|
||||
)
|
||||
|
||||
func TestBuiltinIDGenerators(t *testing.T) {
|
||||
@@ -46,7 +41,6 @@ func TestBuiltinSamplesReproducible(t *testing.T) {
|
||||
`"{uuid()}"`, `"{ulid()}"`,
|
||||
`"{nanoid(12)}"`, `"{int(1,1000000)}"`,
|
||||
`"{float(0,1,6)}"`, `"{base64(12)}"`, `"{iban(SE)}"`,
|
||||
`"{date(2000-01-01,2020-12-31,'2006-01-02 15:04:05')}"`, `"{time('15:04:05')}"`,
|
||||
} {
|
||||
if a, b := mustRender(t, engine(7), tmpl), mustRender(t, engine(7), tmpl); a != b {
|
||||
t.Fatalf("%s not reproducible: %q != %q", tmpl, a, b)
|
||||
@@ -75,9 +69,6 @@ func TestBuiltinFloat(t *testing.T) {
|
||||
if got := mustRender(t, f, `"{float(1,2,3)}"`); !re.MatchString(got) {
|
||||
t.Fatalf("float(1,2,3) = %q, want d.ddd in [1,2]", got)
|
||||
}
|
||||
if got := mustRender(t, f, `"{float(-1,1,0)}"`); got == "-0" {
|
||||
t.Fatalf("float(-1,1,0) = %q, want a zero printed unsigned", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -104,10 +95,10 @@ func TestBuiltinChecksums(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuiltinSeqPerGenerator pins seq's contract: a counter from 1, advancing on
|
||||
// TestBuiltinSeqPerSession pins seq's contract: a counter from 1, advancing on
|
||||
// each call, named counters independent, and the whole thing scoped to one
|
||||
// draw state so a fresh Generator restarts at 1.
|
||||
func TestBuiltinSeqPerGenerator(t *testing.T) {
|
||||
// session so a fresh Generator restarts at 1.
|
||||
func TestBuiltinSeqPerSession(t *testing.T) {
|
||||
f := engine(1)
|
||||
for i := 1; i <= 5; i++ {
|
||||
if got := mustRender(t, f, `"{seq()}"`); got != strconv.Itoa(i) {
|
||||
@@ -124,9 +115,9 @@ func TestBuiltinSeqPerGenerator(t *testing.T) {
|
||||
if got := mustRender(t, engine(2), `{"format":"{seq()}","repeat":3,"separator":","}`); got != "1,2,3" {
|
||||
t.Fatalf("repeat seq = %q, want 1,2,3", got)
|
||||
}
|
||||
// ...and a fresh generator restarts from 1.
|
||||
// ...and a fresh session restarts from 1.
|
||||
if got := mustRender(t, engine(2), `"{seq()}"`); got != "1" {
|
||||
t.Fatalf("new generator seq = %q, want 1", got)
|
||||
t.Fatalf("new session seq = %q, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,7 +136,7 @@ func TestBuiltinArgLimits(t *testing.T) {
|
||||
`{"format":"{x}","x":"1","repeat":2000000000}`, // repeat -> multi-GB join
|
||||
}
|
||||
for _, tmpl := range bad {
|
||||
if _, err := resolved(t, tmpl); err == nil {
|
||||
if _, err := compile(parse(t, tmpl)); err == nil {
|
||||
t.Errorf("compile(%s) = nil error, want a limit rejection", tmpl)
|
||||
}
|
||||
}
|
||||
@@ -190,229 +181,52 @@ func ibanValid(s string) bool {
|
||||
return rem == 1
|
||||
}
|
||||
|
||||
// mustPanic fails unless call panics with invariant.Broken's phrase, which is what
|
||||
// separates a reported invariant break from a silently wrong value.
|
||||
// TestRegistryShapes pins the builtin contract compileOps relies on: every entry
|
||||
// supplies prep, and args parsed at compile only behind a check.
|
||||
func TestRegistryShapes(t *testing.T) {
|
||||
for name, b := range builtins {
|
||||
if b.prep == nil {
|
||||
t.Errorf("builtin %q has no prep: compileOps would call a nil func", name)
|
||||
}
|
||||
if b.arity != 0 && b.check == nil {
|
||||
t.Errorf("builtin %q parses args in prep with no check", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestArgGuardsPanic pins the guards that report a builtin arg its check should
|
||||
// have rejected. No data reaches them — checkFunc runs a builtin's check before
|
||||
// compileOps ever calls prep — so they are exercised directly.
|
||||
func TestArgGuardsPanic(t *testing.T) {
|
||||
for name, call := range map[string]func(){
|
||||
"atoi on an unvalidated arg": func() { atoi("nope") },
|
||||
"atof on an unvalidated arg": func() { atof("nope") },
|
||||
} {
|
||||
mustPanic(t, name, call)
|
||||
}
|
||||
}
|
||||
|
||||
// mustPanic fails unless call panics, which is what separates a reported
|
||||
// invariant break from a silently wrong value.
|
||||
func mustPanic(t *testing.T, name string, call func()) {
|
||||
t.Helper()
|
||||
defer func() {
|
||||
if r := recover(); r == nil || !strings.HasPrefix(fmt.Sprint(r), invariant.Broken("")) {
|
||||
t.Errorf("%s: recovered %v, want the invariant reported as an internal error", name, r)
|
||||
if recover() == nil {
|
||||
t.Errorf("%s: no panic, want the invariant reported", name)
|
||||
}
|
||||
}()
|
||||
call()
|
||||
}
|
||||
|
||||
func TestClassBuiltinArgs(t *testing.T) {
|
||||
for _, bad := range []string{`"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} {
|
||||
if _, err := resolved(t, bad); err == nil {
|
||||
for _, bad := range []string{`"{digits(0)}"`, `"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} {
|
||||
if _, err := compile(parse(t, bad)); err == nil {
|
||||
t.Errorf("compile(%s) = nil error, want the arg rejected", bad)
|
||||
}
|
||||
}
|
||||
for _, ok := range []string{`"{digits(1048576)}"`, `"{upper(1)}"`, `"{lower(26)}"`} {
|
||||
if _, err := resolved(t, ok); err != nil {
|
||||
if _, err := compile(parse(t, ok)); err != nil {
|
||||
t.Errorf("compile(%s) = %v", ok, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuiltinDateAndTime pins the two clock-free samples: date draws a second in
|
||||
// [from 00:00:00, to 23:59:59], both days reachable, and renders it in the quoted Go
|
||||
// layout, commas and English names included; time draws a second within one day.
|
||||
func TestBuiltinDateAndTime(t *testing.T) {
|
||||
f := engine(1)
|
||||
seen := map[string]bool{}
|
||||
for i := 0; i < 500; i++ {
|
||||
got := mustRender(t, f, `"{date(1990-01-01,1990-12-31,'2006-01-02')}"`)
|
||||
if d, err := time.Parse("2006-01-02", got); err != nil || d.Year() != 1990 {
|
||||
t.Fatalf("date = %q, want a 1990 calendar date (err %v)", got, err)
|
||||
}
|
||||
seen[got] = true
|
||||
}
|
||||
if len(seen) < 200 {
|
||||
t.Fatalf("date drew %d distinct days of 365 in 500, want a uniform spread", len(seen))
|
||||
}
|
||||
lo, hi := false, false
|
||||
for i := 0; i < 200; i++ {
|
||||
got := mustRender(t, f, `"{date(2020-02-28,2020-02-29,'2006-01-02')}"`)
|
||||
if got != "2020-02-28" && got != "2020-02-29" {
|
||||
t.Fatalf("date(2020-02-28,2020-02-29) = %q, out of range", got)
|
||||
}
|
||||
lo, hi = lo || got == "2020-02-28", hi || got == "2020-02-29"
|
||||
}
|
||||
if !lo || !hi {
|
||||
t.Fatalf("date never hit a bound: lo=%v hi=%v (bounds must be inclusive)", lo, hi)
|
||||
}
|
||||
if got := mustRender(t, f, `"{date(2020-07-04,2020-07-05,'January 2, 2006')}"`); got != "July 4, 2020" && got != "July 5, 2020" {
|
||||
t.Fatalf("date with a comma in its layout = %q", got)
|
||||
}
|
||||
rfc := regexp.MustCompile(`^2021-\d\d-\d\dT\d\d:\d\d:\d\dZ$`)
|
||||
clock := regexp.MustCompile(`^([01]\d|2[0-3]):[0-5]\d$`)
|
||||
ampm := regexp.MustCompile(`^(1[0-2]|[1-9]):[0-5]\d (AM|PM)$`)
|
||||
seconds := map[string]bool{}
|
||||
for i := 0; i < 300; i++ {
|
||||
got := mustRender(t, f, `"{date(2021-01-01,2021-12-31,'2006-01-02T15:04:05Z07:00')}"`)
|
||||
if !rfc.MatchString(got) {
|
||||
t.Fatalf("date in an RFC 3339 layout = %q, want %s", got, rfc)
|
||||
}
|
||||
seconds[got[17:19]] = true
|
||||
if got := mustRender(t, f, `"{time('15:04')}"`); !clock.MatchString(got) {
|
||||
t.Fatalf("time('15:04') = %q, want %s", got, clock)
|
||||
}
|
||||
if got := mustRender(t, f, `"{time('3:04 PM')}"`); !ampm.MatchString(got) {
|
||||
t.Fatalf("time('3:04 PM') = %q, want %s", got, ampm)
|
||||
}
|
||||
got = mustRender(t, f, `"{date(1950-01-01,2000-12-31,'060102')}-238{luhn()}"`)
|
||||
if d := digitsOnly(got); len(d) != 10 || !luhnValid(d) {
|
||||
t.Fatalf("a personnummer over date() = %q, want ten Luhn-valid digits", got)
|
||||
}
|
||||
}
|
||||
if len(seconds) < 30 {
|
||||
t.Fatalf("date drew %d distinct seconds in 300, want the whole day, not midnight", len(seconds))
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuiltinDateArgs pins the New-time checks: bounds are calendar dates in order, and the
|
||||
// layout is quoted.
|
||||
func TestBuiltinDateArgs(t *testing.T) {
|
||||
for tmpl, want := range map[string]string{
|
||||
`"{date(1990-13-01,1990-12-31,'2006-01-02')}"`: "1990-13-01",
|
||||
`"{time('2006-01-02 15:04')}"`: "write date(from,to,layout)",
|
||||
`"{date(1990-12-31,1990-01-01,'2006-01-02')}"`: "is after",
|
||||
`"{date(1990-01-01,1990-12-31,2006-01-02)}"`: "'2006-01-02'",
|
||||
`"{date(1990-01-01,1990-12-31,'January 2, 2006)}"`: "'",
|
||||
`"{date(1990-01-01,1990-12-31,\"January 2, 2006\")}"`: "quoted: 'January 2, 2006'",
|
||||
`"{date(1990-01-01,1990-12-31)}"`: "3 arguments",
|
||||
`"{date(1990-01-01,1990-12-31,January 2, 2006)}"`: "'January 2, 2006'",
|
||||
`"{time(3:04 PM, Mon)}"`: "'3:04 PM, Mon'",
|
||||
`"{time(15:04)}"`: "'15:04'",
|
||||
} {
|
||||
_, err := resolved(t, tmpl)
|
||||
if err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("compile(%s) = %v, want an error mentioning %q", tmpl, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuiltinLayoutErrorsNameARunnableSpelling pins the two layout errors a user
|
||||
// can follow: a double-quoted layout is named single-quoted, without its own
|
||||
// quotes carried into the suggestion, whether it split on a comma or not.
|
||||
func TestBuiltinLayoutErrorsNameARunnableSpelling(t *testing.T) {
|
||||
_, err := compile(parse(t, `"{date(1990-01-01,1990-12-31,\"2006-01-02\")}"`))
|
||||
if err == nil || !strings.Contains(err.Error(), "write '2006-01-02'") {
|
||||
t.Fatalf("a double-quoted layout = %v, want it named single-quoted", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), `'"`) {
|
||||
t.Errorf("%v names a layout that renders its own quotes", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "is double-quoted") {
|
||||
t.Errorf("%v does not say which quotes were wrong", err)
|
||||
}
|
||||
_, err = compile(parse(t, `"{time(15:04)}"`))
|
||||
if err == nil || !strings.Contains(err.Error(), "write '15:04'") {
|
||||
t.Fatalf("a bare layout = %v, want it named quoted", err)
|
||||
}
|
||||
// The comma hint belongs to a layout that split, not to a call given extra args.
|
||||
_, err = compile(parse(t, `"{time(0,12,'15:04')}"`))
|
||||
if err == nil || !strings.Contains(err.Error(), "takes 1 argument, got 3") {
|
||||
t.Fatalf("time with three args = %v, want the count named in the singular", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "holding a comma") {
|
||||
t.Errorf("%v offers the comma hint though the layout is already quoted", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuiltinDateSpansOneDay pins that from == to is a day: with a clock layout it
|
||||
// draws every second of it, and without one it prints that day.
|
||||
func TestBuiltinDateSpansOneDay(t *testing.T) {
|
||||
f := engine(1)
|
||||
seen := map[string]bool{}
|
||||
for i := 0; i < 500; i++ {
|
||||
got := mustRender(t, f, `"{date(2026-01-01,2026-01-01,'2006-01-02 15:04:05')}"`)
|
||||
if !strings.HasPrefix(got, "2026-01-01 ") {
|
||||
t.Fatalf("date over one day = %q, out of range", got)
|
||||
}
|
||||
seen[got] = true
|
||||
}
|
||||
if len(seen) < 400 {
|
||||
t.Fatalf("date over one day drew %d distinct seconds in 500, want the whole day", len(seen))
|
||||
}
|
||||
if got := mustRender(t, f, `"{date(2026-01-01,2026-01-01,'2006-01-02')}"`); got != "2026-01-01" {
|
||||
t.Fatalf("one day in a date-only layout = %q, want that day", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassBuiltins(t *testing.T) {
|
||||
cases := map[string]*regexp.Regexp{
|
||||
`"{digits(1)}"`: regexp.MustCompile(`^[0-9]$`),
|
||||
`"{digits(3)}"`: regexp.MustCompile(`^[0-9]{3}$`),
|
||||
`"{int(1,9)}"`: regexp.MustCompile(`^[1-9]$`),
|
||||
`"{upper(1)}"`: regexp.MustCompile(`^[A-Z]$`),
|
||||
`"{lower(1)}"`: regexp.MustCompile(`^[a-z]$`),
|
||||
`"{upper(2)}{lower(2)}"`: regexp.MustCompile(`^[A-Z]{2}[a-z]{2}$`),
|
||||
}
|
||||
f := engine(7)
|
||||
for tmpl, re := range cases {
|
||||
for i := 0; i < 100; i++ {
|
||||
if got := mustRender(t, f, tmpl); !re.MatchString(got) {
|
||||
t.Fatalf("%s produced %q, want %s", tmpl, got, re)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFunctionTokenLuhn(t *testing.T) {
|
||||
// {luhn()} appends a Luhn check digit over the digits emitted so far in the
|
||||
// current expansion (non-digits skipped but kept). It reads the output
|
||||
// buffer, so a value is never re-rendered. Bodies are escaped to fix input.
|
||||
f := engine(1)
|
||||
cases := map[string]string{
|
||||
`"811218987{luhn()}"`: "8112189876", // personnummer body
|
||||
`"7992739871{luhn()}"`: "79927398713", // classic Luhn vector
|
||||
`"811218-987{luhn()}"`: "811218-9876", // '-' skipped, kept
|
||||
`{"format":"{n}{luhn()}","n":"811218987"}`: "8112189876", // over a rendered token
|
||||
}
|
||||
for tmpl, want := range cases {
|
||||
if got := mustRender(t, f, tmpl); got != want {
|
||||
t.Fatalf("%s = %q, want %q", tmpl, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltinCompileErrors(t *testing.T) {
|
||||
for _, bad := range []string{
|
||||
`"{upper(x)}"`, // count must be an integer
|
||||
`"{int(a,b)}"`, // non-integer args
|
||||
`"{int(5,1)}"`, // min > max
|
||||
`"{nanoid(-1)}"`, // negative count
|
||||
`"{float(1,2,-1)}"`, // negative decimals
|
||||
`"{float(NaN,NaN,2)}"`, // bounds must be finite
|
||||
`"{float(Inf,Inf,2)}"`, // same-sign infinities
|
||||
`"{float(1,NaN,2)}"`, // one NaN bound
|
||||
`"{float(-Inf,1,2)}"`, // one infinite bound
|
||||
`"{iban(US)}"`, // unsupported country
|
||||
`"{seq(a,b)}"`, // seq takes at most one name
|
||||
} {
|
||||
if _, err := resolved(t, bad); err == nil {
|
||||
t.Errorf("compile(%s) = nil error, want error", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestArgErrorsNameTheSpelling(t *testing.T) {
|
||||
if _, err := resolved(t, `"{hex(99999999999999999999)}"`); err == nil || !strings.Contains(err.Error(), "exceeds the maximum") {
|
||||
t.Errorf("compile(hex past the range) = %v, want the maximum named", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASignOrLeadingZeroIsTheInteger(t *testing.T) {
|
||||
for long, short := range map[string]string{
|
||||
`"{float(1,2,02)}"`: `"{float(1,2,2)}"`,
|
||||
`{"format":"{calc(a,02)}","a":"1"}`: `{"format":"{calc(a,2)}","a":"1"}`,
|
||||
`"{digits(+5)}"`: `"{digits(5)}"`,
|
||||
`"{digits(05)}"`: `"{digits(5)}"`,
|
||||
`"{int(007,9)}"`: `"{int(7,9)}"`,
|
||||
`"{int(+1,5)}"`: `"{int(1,5)}"`,
|
||||
} {
|
||||
sameRenders(t, long, short)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,149 +2,317 @@ package fejkdata
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/larvit/fejkdata/internal/builtinfunc"
|
||||
"github.com/larvit/fejkdata/internal/grammar"
|
||||
"github.com/larvit/fejkdata/internal/invariant"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// checkCalcFields runs checkOperands over each calc operand naming a field; an operand naming no
|
||||
// field is a name, which checkCalcNames checks once names resolve.
|
||||
func checkCalcFields(args []string, fields map[string]node) error {
|
||||
return checkOperands(args[0], builtinfunc.ParsedCalc(args[0]), func(name string) []node {
|
||||
if n, ok := fields[name]; ok {
|
||||
return []node{n}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// calcNode is a parsed expression node. It evaluates over the operand values expand
|
||||
// read, so the evaluator touches neither the rng nor the node tree.
|
||||
type calcNode interface {
|
||||
eval(operands []string) float64
|
||||
}
|
||||
|
||||
// checkCalcNames holds each calc of t reading a name to the checks checkCalcFields makes of a field.
|
||||
func checkCalcNames(label string, t *template) error {
|
||||
for _, o := range t.compiled.ops {
|
||||
if o.Fn != builtinfunc.CalcName || !slices.ContainsFunc(o.operands, func(a arm) bool { return a.kind == namedRead }) {
|
||||
continue
|
||||
type calcNum float64 // a number literal
|
||||
type calcVar string // a sibling-field name, before indexVars places it
|
||||
type calcIdx int // an operand, by its position in the values expand read
|
||||
type calcNeg struct{ x calcNode }
|
||||
type calcBin struct { // a + - * / b
|
||||
op byte
|
||||
l, r calcNode
|
||||
}
|
||||
|
||||
func (n calcNum) eval([]string) float64 { return float64(n) }
|
||||
|
||||
func (n calcIdx) eval(operands []string) float64 {
|
||||
v, err := strconv.ParseFloat(strings.TrimSpace(operands[n]), 64)
|
||||
if err != nil {
|
||||
return math.NaN() // a non-numeric operand stays visible, never an error
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// eval on an unplaced name cannot happen: calcPrep runs indexVars over every
|
||||
// expression it compiles, so only a calcIdx reaches a render. It panics rather
|
||||
// than returning NaN, so a node kind indexVars forgets is a stack trace and not a
|
||||
// silently wrong number.
|
||||
func (n calcVar) eval([]string) float64 {
|
||||
panic(fmt.Sprintf("fejkdata: calc operand %q was never placed", string(n)))
|
||||
}
|
||||
|
||||
func (n calcNeg) eval(operands []string) float64 { return -n.x.eval(operands) }
|
||||
|
||||
func (n calcBin) eval(operands []string) float64 {
|
||||
l, r := n.l.eval(operands), n.r.eval(operands)
|
||||
switch n.op {
|
||||
case '+':
|
||||
return l + r
|
||||
case '-':
|
||||
return l - r
|
||||
case '*':
|
||||
return l * r
|
||||
default: // '/'
|
||||
return l / r
|
||||
}
|
||||
}
|
||||
|
||||
// checkCalc validates a calc token at compile time: a parseable expression whose
|
||||
// operands all name existing fields, and an optional non-negative integer dp. It
|
||||
// is a builtin check (fields first), so calc dispatches through the registry like
|
||||
// every other {name(args)} function.
|
||||
func checkCalc(fields map[string]node, args []string) error {
|
||||
if len(args) < 1 || len(args) > 2 {
|
||||
return fmt.Errorf("calc takes an expression and an optional decimals count, got %d args", len(args))
|
||||
}
|
||||
expr, err := parseCalc(args[0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("calc(%q): %w", args[0], err)
|
||||
}
|
||||
for _, name := range calcVars(expr) {
|
||||
operand, ok := fields[name]
|
||||
if !ok {
|
||||
return fmt.Errorf("calc(%q): no field %q", args[0], name)
|
||||
}
|
||||
if err := checkOperands(o.Args[0], builtinfunc.ParsedCalc(o.Args[0]), operandNodes(o)); err != nil {
|
||||
return fmt.Errorf("%s: token {%s}: %w", label, o.Body, err)
|
||||
if text, never := neverNumeric(operand); never {
|
||||
return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", args[0], name, text)
|
||||
}
|
||||
}
|
||||
if len(args) == 2 {
|
||||
if dp, err := strconv.Atoi(args[1]); err != nil || dp < 0 || dp > maxDecimals {
|
||||
return fmt.Errorf("calc decimals %q must be an integer in 0..%d", args[1], maxDecimals)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// operandNodes lists every node each operand of o may render, once its template and names resolve.
|
||||
func operandNodes(o op) func(name string) []node {
|
||||
return func(name string) []node {
|
||||
a := o.operands[slices.IndexFunc(o.operands, func(a arm) bool { return a.head == name })]
|
||||
if len(a.leaves) == 0 {
|
||||
panic(invariant.Broken("calc operand %q was read before it was compiled", name))
|
||||
}
|
||||
return a.leaves
|
||||
}
|
||||
}
|
||||
|
||||
// checkOperands refuses an operand that is never a number and a division by a constant zero.
|
||||
// operand lists every node an operand may render, nil while that is unknown.
|
||||
func checkOperands(text string, c grammar.Calc, operand func(name string) []node) error {
|
||||
for _, name := range c.Operands {
|
||||
if rendered, never := allNeverNumeric(operand(name)); never {
|
||||
return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", text, name, rendered)
|
||||
}
|
||||
}
|
||||
if divisor, zero := constantZeroDivisor(c.Expr, operand); zero {
|
||||
return fmt.Errorf("calc(%q) divides by %s, which is always zero", text, divisor)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// constantZeroDivisor finds a division whose right side is a constant zero: number
|
||||
// literals and fixed operands folded, anything that varies left unknown.
|
||||
func constantZeroDivisor(n grammar.CalcNode, operand func(string) []node) (string, bool) {
|
||||
switch n := n.(type) {
|
||||
case grammar.CalcNeg:
|
||||
return constantZeroDivisor(n.X, operand)
|
||||
case grammar.CalcBin:
|
||||
if n.Operator == '/' {
|
||||
if v, known := constantValue(n.R, operand); known && v == 0 {
|
||||
return grammar.CalcText(n.R), true
|
||||
}
|
||||
}
|
||||
if d, zero := constantZeroDivisor(n.L, operand); zero {
|
||||
return d, true
|
||||
}
|
||||
return constantZeroDivisor(n.R, operand)
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// constantValue evaluates an expression whose every operand is fixed.
|
||||
func constantValue(n grammar.CalcNode, operand func(string) []node) (float64, bool) {
|
||||
switch n := n.(type) {
|
||||
case grammar.CalcNum:
|
||||
return float64(n), true
|
||||
case grammar.CalcVar:
|
||||
nodes := operand(n.Name)
|
||||
if len(nodes) != 1 {
|
||||
return 0, false
|
||||
}
|
||||
t, ok := nodes[0].(*template)
|
||||
if !ok || t.repeat != 1 {
|
||||
return 0, false
|
||||
}
|
||||
lit, fixed := t.fixedText()
|
||||
if !fixed {
|
||||
return 0, false
|
||||
}
|
||||
v, err := strconv.ParseFloat(strings.TrimSpace(lit), 64)
|
||||
return v, err == nil
|
||||
case grammar.CalcNeg:
|
||||
v, ok := constantValue(n.X, operand)
|
||||
return -v, ok
|
||||
case grammar.CalcBin:
|
||||
l, lok := constantValue(n.L, operand)
|
||||
r, rok := constantValue(n.R, operand)
|
||||
if !lok || !rok {
|
||||
return 0, false
|
||||
}
|
||||
return builtinfunc.Arith(n.Operator, l, r), true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// neverNumeric reports a node no render of which is a number: a null, a repeat of 0, fixed
|
||||
// text that does not parse, or a choice of only such items. text is one such render.
|
||||
// neverNumeric reports a node no render of which is a number: fixed text that does
|
||||
// not parse, or a choice of only such items. text is one such render.
|
||||
func neverNumeric(n node) (text string, never bool) {
|
||||
switch n := n.(type) {
|
||||
case *nullItem:
|
||||
return "", true
|
||||
case *template:
|
||||
lit, fixed := n.fixedText()
|
||||
if n.repeat == 0 {
|
||||
return "", true
|
||||
}
|
||||
if !fixed || n.repeat != 1 {
|
||||
if !n.fixed || n.repeat > 1 {
|
||||
return "", false
|
||||
}
|
||||
if _, err := strconv.ParseFloat(strings.TrimSpace(lit), 64); err != nil {
|
||||
return lit, true
|
||||
if _, err := strconv.ParseFloat(strings.TrimSpace(n.lit), 64); err != nil {
|
||||
return n.lit, true
|
||||
}
|
||||
case *choice:
|
||||
return allNeverNumeric(n.items)
|
||||
for _, it := range n.items {
|
||||
t, itemNever := neverNumeric(it)
|
||||
if !itemNever {
|
||||
return "", false
|
||||
}
|
||||
text = t
|
||||
}
|
||||
return text, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// allNeverNumeric reports whether no render of any of nodes is a number; false for no nodes.
|
||||
func allNeverNumeric(nodes []node) (text string, never bool) {
|
||||
for _, n := range nodes {
|
||||
t, nodeNever := neverNumeric(n)
|
||||
if !nodeNever {
|
||||
return "", false
|
||||
}
|
||||
text = t
|
||||
// calcPrep parses the expression and decimals once, at compile time, and places each
|
||||
// operand name at the position expand will read it into. checkCalc proved both args
|
||||
// valid, so no step here can fail; dp -1 prints the minimal form.
|
||||
func calcPrep(args []string) callFn {
|
||||
expr, err := parseCalc(args[0])
|
||||
if err != nil { // a nil AST would be a nil dereference per render, with no message
|
||||
panic(fmt.Sprintf("fejkdata: calc(%q) reached prep unparsed: %v", args[0], err))
|
||||
}
|
||||
at := make(map[string]int)
|
||||
for i, name := range calcVars(expr) {
|
||||
at[name] = i
|
||||
}
|
||||
placed := indexVars(expr, at)
|
||||
dp := -1
|
||||
if len(args) == 2 {
|
||||
dp = atoi(args[1])
|
||||
}
|
||||
return func(_ *session, _ string, operands []string) string {
|
||||
return strconv.FormatFloat(placed.eval(operands), 'f', dp, 64)
|
||||
}
|
||||
return text, len(nodes) > 0
|
||||
}
|
||||
|
||||
// indexVars replaces each operand name with its position in the values expand reads.
|
||||
// Both sides take that order from calcVars, so they cannot drift.
|
||||
func indexVars(n calcNode, at map[string]int) calcNode {
|
||||
switch n := n.(type) {
|
||||
case calcVar:
|
||||
i, placed := at[string(n)]
|
||||
if !placed { // calcVars named every operand, so a miss means the two disagree
|
||||
panic(fmt.Sprintf("fejkdata: calc operand %q is not among the names read for it", string(n)))
|
||||
}
|
||||
return calcIdx(i)
|
||||
case calcNeg:
|
||||
return calcNeg{indexVars(n.x, at)}
|
||||
case calcBin:
|
||||
return calcBin{n.op, indexVars(n.l, at), indexVars(n.r, at)}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// calcOperands lists the sibling-field names a calc's args read. checkCalc reports
|
||||
// an expression that does not parse, so one that does not simply names nothing.
|
||||
func calcOperands(args []string) []string {
|
||||
if len(args) == 0 {
|
||||
return nil
|
||||
}
|
||||
expr, err := parseCalc(args[0])
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return calcVars(expr)
|
||||
}
|
||||
|
||||
// calcVars lists the distinct field names an expression reads, in the order it first
|
||||
// names each. That order is the contract between expand, which reads the operands
|
||||
// into a slice, and indexVars, which places each name at its position in it.
|
||||
func calcVars(n calcNode) []string {
|
||||
var out []string
|
||||
seen := map[string]bool{}
|
||||
var walk func(calcNode)
|
||||
walk = func(n calcNode) {
|
||||
switch n := n.(type) {
|
||||
case calcVar:
|
||||
if name := string(n); !seen[name] {
|
||||
seen[name] = true
|
||||
out = append(out, name)
|
||||
}
|
||||
case calcNeg:
|
||||
walk(n.x)
|
||||
case calcBin:
|
||||
walk(n.l)
|
||||
walk(n.r)
|
||||
}
|
||||
}
|
||||
walk(n)
|
||||
return out
|
||||
}
|
||||
|
||||
// calcParser is a recursive-descent parser over the expression runes, threading
|
||||
// expr -> term -> factor for the standard * / before + - precedence.
|
||||
type calcParser struct {
|
||||
rs []rune
|
||||
pos int
|
||||
}
|
||||
|
||||
// parseCalc parses a whole expression, requiring it to consume all input.
|
||||
func parseCalc(expr string) (calcNode, error) {
|
||||
p := &calcParser{rs: []rune(expr)}
|
||||
if p.space(); p.pos >= len(p.rs) {
|
||||
return nil, fmt.Errorf("empty expression")
|
||||
}
|
||||
n, err := p.expr()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p.space(); p.pos != len(p.rs) {
|
||||
return nil, fmt.Errorf("unexpected %q", string(p.rs[p.pos:]))
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (p *calcParser) space() {
|
||||
for p.pos < len(p.rs) && unicode.IsSpace(p.rs[p.pos]) {
|
||||
p.pos++
|
||||
}
|
||||
}
|
||||
|
||||
func (p *calcParser) expr() (calcNode, error) { return p.binary(p.term, '+', '-') }
|
||||
func (p *calcParser) term() (calcNode, error) { return p.binary(p.factor, '*', '/') }
|
||||
|
||||
// binary parses a left-associative run of next() operands joined by the given
|
||||
// operators, the one shape expr and term share.
|
||||
func (p *calcParser) binary(next func() (calcNode, error), ops ...byte) (calcNode, error) {
|
||||
n, err := next()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for {
|
||||
p.space()
|
||||
if p.pos >= len(p.rs) || !contains(ops, byte(p.rs[p.pos])) {
|
||||
return n, nil
|
||||
}
|
||||
op := byte(p.rs[p.pos])
|
||||
p.pos++
|
||||
r, err := next()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n = calcBin{op, n, r}
|
||||
}
|
||||
}
|
||||
|
||||
// factor is a table-shaped scanner, one case per token kind, kept whole on purpose.
|
||||
func (p *calcParser) factor() (calcNode, error) {
|
||||
p.space()
|
||||
if p.pos >= len(p.rs) {
|
||||
return nil, fmt.Errorf("unexpected end of expression")
|
||||
}
|
||||
switch c := p.rs[p.pos]; {
|
||||
case c == '-':
|
||||
p.pos++
|
||||
x, err := p.factor()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return calcNeg{x}, nil
|
||||
case c == '(':
|
||||
p.pos++
|
||||
n, err := p.expr()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p.space(); p.pos >= len(p.rs) || p.rs[p.pos] != ')' {
|
||||
return nil, fmt.Errorf("missing ')'")
|
||||
}
|
||||
p.pos++
|
||||
return n, nil
|
||||
case c == '.' || c >= '0' && c <= '9':
|
||||
return p.number()
|
||||
case c == '_' || unicode.IsLetter(c):
|
||||
return p.ident()
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected %q", string(c))
|
||||
}
|
||||
}
|
||||
|
||||
func (p *calcParser) number() (calcNode, error) {
|
||||
start, dot := p.pos, false
|
||||
for p.pos < len(p.rs) {
|
||||
if c := p.rs[p.pos]; c >= '0' && c <= '9' {
|
||||
p.pos++
|
||||
} else if c == '.' && !dot {
|
||||
dot, p.pos = true, p.pos+1
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
v, err := strconv.ParseFloat(string(p.rs[start:p.pos]), 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bad number %q", string(p.rs[start:p.pos]))
|
||||
}
|
||||
return calcNum(v), nil
|
||||
}
|
||||
|
||||
// ident reads a field name: a letter or '_', then letters, digits or '_'. A '-'
|
||||
// is always the minus operator, so a hyphenated field name can't be an operand.
|
||||
func (p *calcParser) ident() (calcNode, error) {
|
||||
start := p.pos
|
||||
for p.pos < len(p.rs) {
|
||||
if c := p.rs[p.pos]; c == '_' || unicode.IsLetter(c) || unicode.IsDigit(c) {
|
||||
p.pos++
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
return calcVar(string(p.rs[start:p.pos])), nil
|
||||
}
|
||||
|
||||
func contains(bs []byte, b byte) bool {
|
||||
for _, x := range bs {
|
||||
if x == b {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
+61
-164
@@ -3,11 +3,8 @@ package fejkdata
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/larvit/fejkdata/internal/grammar"
|
||||
)
|
||||
|
||||
// TestCalcArithmetic pins the operators, precedence, parentheses and unary minus
|
||||
@@ -39,7 +36,6 @@ func TestCalcAuto(t *testing.T) {
|
||||
`"{calc(10 / 3)}"`: "3.3333333333333335",
|
||||
`"{calc(6 / 2)}"`: "3",
|
||||
`"{calc(1 / 4)}"`: "0.25",
|
||||
`"{calc(0 * -1)}"`: "0",
|
||||
}
|
||||
for tmpl, want := range cases {
|
||||
if got := mustRender(t, f, tmpl); got != want {
|
||||
@@ -56,7 +52,6 @@ func TestCalcDecimals(t *testing.T) {
|
||||
`"{calc(10 / 3, 2)}"`: "3.33",
|
||||
`"{calc(10 / 3, 0)}"`: "3",
|
||||
`"{calc(2 * 3, 2)}"`: "6.00",
|
||||
`"{calc(-0.001, 2)}"`: "0.00",
|
||||
}
|
||||
for tmpl, want := range cases {
|
||||
if got := mustRender(t, f, tmpl); got != want {
|
||||
@@ -75,9 +70,6 @@ func TestCalcFields(t *testing.T) {
|
||||
if got := mustRender(t, engine(1), `{"format":"{calc(a - b)}","a":{"format":"{n}","n":"10"},"b":"3"}`); got != "7" {
|
||||
t.Fatalf("calc(a - b) = %q, want 7", got)
|
||||
}
|
||||
if got := mustRender(t, engine(1), `{"format":"{calc(b * (a - b))}","a":"10","b":"3"}`); got != "21" {
|
||||
t.Fatalf("calc(b * (a - b)) = %q, want 21", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCalcNonNumericIsNaN pins the never-fail rule: a field that sometimes does
|
||||
@@ -100,26 +92,43 @@ func TestCalcReproducible(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestParsedCallReadsOnlyAnOperandBuiltin pins the operands a parsed call carries for a
|
||||
// call that is not a calc, and for one whose expression does not parse: none, either way.
|
||||
func TestParsedCallReadsOnlyAnOperandBuiltin(t *testing.T) {
|
||||
for _, format := range []string{"{luhn()}", "{calc()}", "{calc(1 +)}", "{calc(()}"} {
|
||||
if toks, err := grammar.ParseFormat(format); err != nil || len(toks) != 1 || tokenReads(toks[0]) != nil {
|
||||
t.Errorf("ParseFormat(%q) = %+v, %v, want one call reading no operand", format, toks, err)
|
||||
// TestTokenOperandsReadsOnlyAnOperandBuiltin pins what the helper answers for a body that
|
||||
// is not a calc, and for one whose expression does not parse: nothing, either way.
|
||||
// checkCalc is what reports a bad expression, so the callers that run after it
|
||||
// never meet one — but they must not have to depend on that order to be safe.
|
||||
func TestTokenOperandsReadsOnlyAnOperandBuiltin(t *testing.T) {
|
||||
for _, body := range []string{"plain", "luhn()", "calc()", "calc(1 +)", "calc(()"} {
|
||||
if got := tokenOperands(body); got != nil {
|
||||
t.Errorf("tokenOperands(%q) = %v, want none", body, got)
|
||||
}
|
||||
}
|
||||
if toks, err := grammar.ParseFormat("{calc(net * qty)}"); err != nil || len(toks) != 1 || len(tokenReads(toks[0])) != 2 {
|
||||
t.Errorf("ParseFormat({calc(net * qty)}) = %+v, %v, want both operands", toks, err)
|
||||
if got := tokenOperands("calc(net * qty)"); len(got) != 2 {
|
||||
t.Errorf("tokenOperands(calc(net * qty)) = %v, want both operands", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCalcGuardsPanic pins the guards on calc's own invariants: checkCalc parsed
|
||||
// the expression before prep sees it, and indexVars places every name calcVars
|
||||
// read, so a break in either is reported rather than computed around.
|
||||
func TestCalcGuardsPanic(t *testing.T) {
|
||||
for name, call := range map[string]func(){
|
||||
"prep on an expression that does not parse": func() { calcPrep([]string{"1 +"}) },
|
||||
"an operand name never placed": func() { calcVar("n").eval(nil) },
|
||||
"indexVars on a name it did not read": func() { indexVars(calcVar("n"), map[string]int{}) },
|
||||
} {
|
||||
mustPanic(t, name, call)
|
||||
}
|
||||
}
|
||||
|
||||
// --- one draw, one value: a calc operand reads the expansion's draw ---
|
||||
|
||||
// TestANamedCalcOperandIsTheOneShown pins how a calc computes from the value its format
|
||||
// shows: the format and the calc read one name.
|
||||
func TestANamedCalcOperandIsTheOneShown(t *testing.T) {
|
||||
// TestCalcOperandReadsTheExpansionsDraw pins the one-draw rule for a calc operand.
|
||||
// A field the format renders and a calc reads is drawn once per expansion, so the
|
||||
// operand shown is the operand computed — the correlation the dotted-path rule
|
||||
// already gives a level (see bound_test.go), applied to a plain sibling.
|
||||
func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) {
|
||||
dir := writeData(t, map[string]string{
|
||||
"inv": `{"format":"{net as n}{qty as q}{n} x {q} = {calc(n * q, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
|
||||
"inv": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
|
||||
})
|
||||
f := newGenerator(t, dir, WithSeed(3))
|
||||
for i := 0; i < 300; i++ {
|
||||
@@ -134,23 +143,39 @@ func TestANamedCalcOperandIsTheOneShown(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestACalcOperandDrawsAfresh pins that a calc reading a field draws it as any {…} does.
|
||||
func TestACalcOperandDrawsAfresh(t *testing.T) {
|
||||
dir := writeData(t, map[string]string{"same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`})
|
||||
// TestCalcOperandSharesOneDraw pins the reach of that hold: the draw belongs to the
|
||||
// expansion, not to the calc, so the bare token rendering the same name reads it too.
|
||||
func TestCalcOperandSharesOneDraw(t *testing.T) {
|
||||
dir := writeData(t, map[string]string{
|
||||
"same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`,
|
||||
})
|
||||
f := newGenerator(t, dir, WithSeed(5))
|
||||
for i := 0; i < 200; i++ {
|
||||
if p := strings.Fields(fake(t, f, "same")); p[0] != p[1] {
|
||||
got := fake(t, f, "same")
|
||||
if p := strings.Fields(got); len(p) != 2 || p[0] != p[1] {
|
||||
t.Fatalf("same = %q, want one value twice", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFieldNoCalcReadsDrawsEachTime guards the boundary: only a name a calc reads is
|
||||
// held, so an ordinary {w} {w} still draws twice.
|
||||
func TestFieldNoCalcReadsDrawsEachTime(t *testing.T) {
|
||||
dir := writeData(t, map[string]string{"two": `{"format":"{w} {w}","w":["1","2","3","4","5"]}`})
|
||||
f := newGenerator(t, dir, WithSeed(5))
|
||||
for i := 0; i < 200; i++ {
|
||||
if p := strings.Fields(fake(t, f, "two")); p[0] != p[1] {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("{w} {calc(w)} never differed in 200 draws, want two independent draws")
|
||||
t.Fatal("{w} {w} never differed in 200 draws, want two independent draws")
|
||||
}
|
||||
|
||||
// TestANamedCalcOperandPicksEachIteration pins that a name a repeat binds picks again on each
|
||||
// iteration, staying one value within it.
|
||||
func TestANamedCalcOperandPicksEachIteration(t *testing.T) {
|
||||
// TestCalcHoldIsPerExpansion pins the scope of the hold: each repeat iteration is
|
||||
// its own expansion, so it draws again while staying self-consistent.
|
||||
func TestCalcHoldIsPerExpansion(t *testing.T) {
|
||||
dir := writeData(t, map[string]string{
|
||||
"rep": `{"format":"{n as m}{m}={calc(m * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`,
|
||||
"rep": `{"format":"{n}={calc(n * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`,
|
||||
})
|
||||
f := newGenerator(t, dir, WithSeed(11))
|
||||
varied := false
|
||||
@@ -171,12 +196,12 @@ func TestANamedCalcOperandPicksEachIteration(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestTwoNamesPickApart pins that a nested template's name is a pick of its own: the inner
|
||||
// pair agrees with itself, not with the outer pair.
|
||||
func TestTwoNamesPickApart(t *testing.T) {
|
||||
// TestCalcHoldIsPerTemplate pins that a nested template holds its own: the inner
|
||||
// {v} and its calc agree with each other, not with the outer pair.
|
||||
func TestCalcHoldIsPerTemplate(t *testing.T) {
|
||||
dir := writeData(t, map[string]string{
|
||||
"nest": `{"format":"{v as a}{a}={calc(a * 1)} {inner}","v":["2","3","4","5","6","7","8","9"],
|
||||
"inner":{"format":"{v as b}{b}={calc(b * 1)}","v":["2","3","4","5","6","7","8","9"]}}`,
|
||||
"nest": `{"format":"{v}={calc(v * 1)} {inner}","v":["2","3","4","5","6","7","8","9"],
|
||||
"inner":{"format":"{v}={calc(v * 1)}","v":["2","3","4","5","6","7","8","9"]}}`,
|
||||
})
|
||||
f := newGenerator(t, dir, WithSeed(13))
|
||||
differed := false
|
||||
@@ -204,7 +229,7 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
|
||||
`{"format":"{calc(x * 2)}","x":["a","b"]}`: `"x"`,
|
||||
`{"format":"{calc(x + y)}","x":"1","y":"{{2}}"}`: `"{2}"`,
|
||||
} {
|
||||
_, err := resolved(t, src)
|
||||
_, err := compile(parse(t, src))
|
||||
if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "never a number") {
|
||||
t.Errorf("compile(%s) = %v, want the operand rejected naming %s", src, err, want)
|
||||
}
|
||||
@@ -215,136 +240,8 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
|
||||
`{"format":"{calc(x * 2)}","x":["1","abc"]}`,
|
||||
`{"format":"{calc(x * 2)}","x":"{digits(2)}"}`,
|
||||
} {
|
||||
if _, err := resolved(t, ok); err != nil {
|
||||
if _, err := compile(parse(t, ok)); err != nil {
|
||||
t.Errorf("compile(%s) = %v, want it accepted", ok, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalcConstantZeroDivisorIsRejected(t *testing.T) {
|
||||
for src, want := range map[string]string{
|
||||
`"{calc(1/0)}"`: "divides by 0",
|
||||
`"{calc(2/(1-1))}"`: "divides by (1 - 1)",
|
||||
`{"format":"{calc(x/y)}","x":"1","y":"0"}`: "divides by y",
|
||||
`{"format":"{calc(x/(y*2))}","x":"1","y":" 0 "}`: "divides by (y * 2)",
|
||||
`{"format":"{calc((a/0)+b)}","a":"1","b":"2"}`: "divides by 0",
|
||||
`{"format":"{calc(a/-0)}","a":"1"}`: "divides by -0",
|
||||
} {
|
||||
if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("compile(%s) = %v, want the constant zero divisor rejected naming %q", src, err, want)
|
||||
}
|
||||
}
|
||||
if _, err := compile(parse(t, `{"format":"{calc(a/(b*c))}","a":"1","b":"0","c":["1","2"]}`)); err != nil {
|
||||
t.Errorf("compile(a/(b*c)) = %v, want a divisor that varies accepted", err)
|
||||
}
|
||||
f := engine(1)
|
||||
for i := 0; i < 50; i++ {
|
||||
if got := mustRender(t, f, `{"format":"{calc(x/y)}","x":"1","y":["0","1"]}`); got == "+Inf" {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("a sometimes-zero divisor never printed +Inf in 50 draws")
|
||||
}
|
||||
|
||||
func TestCalcCompileErrors(t *testing.T) {
|
||||
for _, bad := range []string{
|
||||
`"{calc(1/0)}"`, // a constant zero divisor
|
||||
`{"format":"{calc(x/y)}","x":"1","y":"0"}`, // a fixed zero divisor
|
||||
`"{calc()}"`, // calc needs an expression
|
||||
`"{calc(1 +)}"`, // dangling operator
|
||||
`"{calc((1 + 2)}"`, // unbalanced parenthesis
|
||||
`"{calc(1 2)}"`, // two operands, no operator
|
||||
`"{calc(price)}"`, // operand names no field
|
||||
`"{calc(1, 2, 3)}"`, // too many args
|
||||
`"{calc(1, x)}"`, // decimals arg not an integer
|
||||
`"{calc(1, -1)}"`, // decimals negative
|
||||
} {
|
||||
if _, err := resolved(t, bad); err == nil {
|
||||
t.Errorf("compile(%s) = nil error, want error", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalcReadsAName(t *testing.T) {
|
||||
dir := writeData(t, map[string]string{
|
||||
"n": `["2","3","5","7"]`,
|
||||
"card": `"{/n as k}{k} x 2 = {calc(k * 2)}"`,
|
||||
})
|
||||
f := newGenerator(t, dir, WithSeed(1))
|
||||
for i := 0; i < 50; i++ {
|
||||
var k, double int
|
||||
got := fake(t, f, "card")
|
||||
if _, err := fmt.Sscanf(got, "%d x 2 = %d", &k, &double); err != nil || double != k*2 {
|
||||
t.Fatalf("card = %q, want the calc to compute from the pick {k} prints", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalcReadsANameItReadsNowhereElse(t *testing.T) {
|
||||
dir := writeData(t, map[string]string{
|
||||
"n": `["2","3"]`,
|
||||
"card": `"{/n as k}{calc(k * 2)}"`,
|
||||
})
|
||||
f := newGenerator(t, dir, WithSeed(1))
|
||||
if got := fake(t, f, "card"); got != "4" && got != "6" {
|
||||
t.Fatalf("card = %q, want 4 or 6", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalcRefusesABadOperand(t *testing.T) {
|
||||
for card, want := range map[string]string{
|
||||
`"{/word as w}{w}{calc(w * 2)}"`: `operand "w" is never a number: it renders "def"`,
|
||||
`"{/zero as z}{z}{calc(1 / z)}"`: `divides by z, which is always zero`,
|
||||
`"{calc(nope * 2)}"`: `no field "nope"`,
|
||||
`"{/word as w}{w}{calc(nope * 2)}"`: `no field or name "nope"; the names bound here are "w"`,
|
||||
} {
|
||||
dir := writeData(t, map[string]string{
|
||||
"word": `["abc","def"]`,
|
||||
"zero": `"0"`,
|
||||
"card": card,
|
||||
})
|
||||
if _, err := New(withShipped(), WithDataPath(dir), WithSeed(1)); err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("New with card %s = %v, want an error containing %q", card, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestATypedColumnRefusesACalcOverNoDigits(t *testing.T) {
|
||||
_, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{
|
||||
"row": `{"format":"","x":{"format":"{calc(d + 1)}","d":"{digits(0)}","datatype":"integer"}}`,
|
||||
})))
|
||||
if err == nil || !strings.Contains(err.Error(), "{digits(0)} prints nothing, which is no number") {
|
||||
t.Fatalf("New = %v, want a calc over {digits(0)} refused in a typed column", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATypedColumnProvesACalcOverAName(t *testing.T) {
|
||||
dir := writeData(t, map[string]string{
|
||||
"n": `"{int(1,9)}"`,
|
||||
"order": `{"format":"{/n as k}{k}","double":{"format":"{calc(k * 2)}","datatype":"integer"}}`,
|
||||
})
|
||||
f := newGenerator(t, dir, WithSeed(1))
|
||||
r, err := f.FakeRecord("order")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v, err := strconv.Atoi(r.Columns()[0].Value); err != nil || v < 2 || v > 18 || v%2 != 0 {
|
||||
t.Fatalf("double = %q, want an even integer in 2..18", r.Columns()[0].Value)
|
||||
}
|
||||
if !strings.Contains(r.JSON(), `"double":`+r.Columns()[0].Value) {
|
||||
t.Fatalf("JSON = %s, want double written as a number", r.JSON())
|
||||
}
|
||||
bad := writeData(t, map[string]string{
|
||||
"word": `["1","x"]`,
|
||||
"order": `{"format":"{/word as w}{w}","double":{"format":"{calc(w * 2)}","datatype":"integer"}}`,
|
||||
})
|
||||
if _, err := New(withShipped(), WithDataPath(bad), WithSeed(1)); err == nil || !strings.Contains(err.Error(), `operand "w": "x" is not a number`) {
|
||||
t.Fatalf("New = %v, want the typed column refused over an operand that is not always a number", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARepeatOfZeroIsNeverANumber(t *testing.T) {
|
||||
if _, err := resolved(t, `{"format":"{calc(x / y)}","x":"1","y":{"format":"0","repeat":0}}`); err == nil || !strings.Contains(err.Error(), `operand "y" is never a number: it renders ""`) {
|
||||
t.Errorf("a calc over a repeat of 0 = %v, want the operand refused as never a number", err)
|
||||
}
|
||||
}
|
||||
|
||||
-706
@@ -1,706 +0,0 @@
|
||||
// Package cli is the fejkdata command: it renders the template on its stdin with fake
|
||||
// values from the modules it is given and any directories layered over them. A program
|
||||
// passing its own modules to Main is a fejkdata CLI carrying them, for example
|
||||
// os.Exit(cli.Main(append(data.Modules(), mydata)...)).
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"math/rand/v2"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime/debug"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/larvit/fejkdata"
|
||||
)
|
||||
|
||||
const usage = `Usage: <template> | COMMAND [flags]
|
||||
COMMAND --list | --help | --version
|
||||
|
||||
COMMAND reads one template from stdin and renders it:
|
||||
|
||||
echo 'id: {int(1,999)}' | COMMAND
|
||||
echo '{"format":"{x}","x":["bosse","lina"]}' | COMMAND
|
||||
COMMAND < template.txt
|
||||
|
||||
COMMAND <<'EOF'
|
||||
born {date(1990-01-01,2010-12-31,'2006-01-02')}
|
||||
EOF
|
||||
|
||||
A template is a format string, or a JSON object, array, string or null. Its {…} tokens
|
||||
reach the data by reference from the root: {/path}, for any path --list prints, shipped
|
||||
or layered with --data-path. A reference names a category or a dotted path into one, a
|
||||
table's row by key or name ({/table[KEY]}), and .. steps up from a row to the row it
|
||||
links to.
|
||||
|
||||
Under --format text, the default, what you send is what renders, and COMMAND adds
|
||||
nothing: a format string keeps every byte, so echo's newline ends each render, and
|
||||
printf '%s' sends none. -n joins renders with --separator, empty by default. A JSON
|
||||
object, array or string drops the whitespace around it, and null alone renders nothing;
|
||||
42 or true alone is a format string, printed as written. An empty or blank stdin is a
|
||||
format string like any other, and renders as sent. A quoted heredoc, <<'EOF', passes $,
|
||||
a backslash or a quote as written.
|
||||
|
||||
With --format json, ndjson, csv or sql the template must be a record — a JSON
|
||||
template whose fields are its columns, or a lone reference such as {/path}:
|
||||
one reference and nothing else, with one newline after it allowed — and the rows are
|
||||
written as one JSON array, one JSON object per line, one CSV row (after a header), or
|
||||
one INSERT.
|
||||
|
||||
-d, --data-path D a data directory to layer over the shipped data (repeatable; last wins on a clash)
|
||||
--format F output form: text (default), json, ndjson, csv or sql
|
||||
-h, --help print this help, then exit
|
||||
--list list the paths the data offers, each written after {/ in a template, then exit;
|
||||
--repeat, --separator, --format and --table do nothing here
|
||||
--no-shipped-data load only the --data-path directories
|
||||
-n, --repeat N render the template N times, 0..1048576 (default 1); 0 prints nothing,
|
||||
[] under --format json, or the header under --format csv
|
||||
-s, --seed N same seed, version and data: identical output (default: a random seed)
|
||||
--separator S string between repeated renders (default empty); refused with a record
|
||||
--format, which has its own separators
|
||||
--table T the INSERT target for --format sql, ignored under any other
|
||||
(default: a lone reference's last segment, else records)
|
||||
--version print the version, then exit
|
||||
|
||||
A short flag's value attaches or follows (-n3, -n 3); short flags bundle (-hn 3);
|
||||
-- ends the flags.
|
||||
|
||||
Exit codes: 0 success, 1 runtime error, 2 misuse.
|
||||
`
|
||||
|
||||
type invocation struct {
|
||||
args []string
|
||||
dirs []string
|
||||
format string
|
||||
help bool
|
||||
list bool
|
||||
noShipped bool
|
||||
repeat int
|
||||
seed uint64
|
||||
seeded bool
|
||||
separator string
|
||||
separatorSet bool
|
||||
table string
|
||||
tableSet bool
|
||||
version bool
|
||||
}
|
||||
|
||||
type flagDef struct {
|
||||
long string
|
||||
short string
|
||||
value bool
|
||||
set func(*invocation, string) error
|
||||
}
|
||||
|
||||
var flagDefs = []flagDef{
|
||||
{"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }},
|
||||
{"format", "", true, func(in *invocation, v string) error { in.format = v; return nil }},
|
||||
{"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }},
|
||||
{"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }},
|
||||
{"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }},
|
||||
{"repeat", "n", true, func(in *invocation, v string) error {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 0 || n > fejkdata.MaxRepeat {
|
||||
return fmt.Errorf("--repeat needs an integer in 0..%d, got %q", fejkdata.MaxRepeat, v)
|
||||
}
|
||||
in.repeat = n
|
||||
return nil
|
||||
}},
|
||||
{"seed", "s", true, func(in *invocation, v string) error {
|
||||
n, err := strconv.ParseUint(v, 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("--seed needs an unsigned integer, got %q", v)
|
||||
}
|
||||
in.seed, in.seeded = n, true
|
||||
return nil
|
||||
}},
|
||||
{"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }},
|
||||
{"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }},
|
||||
{"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }},
|
||||
}
|
||||
|
||||
func flagByLong(name string) *flagDef {
|
||||
for i := range flagDefs {
|
||||
if flagDefs[i].long == name {
|
||||
return &flagDefs[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func flagByShort(name string) *flagDef {
|
||||
for i := range flagDefs {
|
||||
if flagDefs[i].short == name {
|
||||
return &flagDefs[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// flagArg is one flag as written: its definition and the value attached to it,
|
||||
// if any.
|
||||
type flagArg struct {
|
||||
def *flagDef
|
||||
value string
|
||||
inline bool
|
||||
}
|
||||
|
||||
// splitFlags reads one argv element as flags: --name or --name=value, or -abc
|
||||
// where each letter is a short flag and the first that takes a value takes the
|
||||
// rest of the element.
|
||||
func splitFlags(arg string) ([]flagArg, error) {
|
||||
if strings.HasPrefix(arg, "--") {
|
||||
name, value, inline := strings.Cut(arg[2:], "=")
|
||||
def := flagByLong(name)
|
||||
if def == nil {
|
||||
return nil, fmt.Errorf("unknown flag --%s", name)
|
||||
}
|
||||
return []flagArg{{def, value, inline}}, nil
|
||||
}
|
||||
if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil {
|
||||
return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name)
|
||||
}
|
||||
var flags []flagArg
|
||||
letters := arg[1:]
|
||||
for i, r := range letters {
|
||||
letter := string(r)
|
||||
def := flagByShort(letter)
|
||||
if def == nil {
|
||||
return nil, fmt.Errorf("unknown flag -%s", letter)
|
||||
}
|
||||
if !def.value {
|
||||
flags = append(flags, flagArg{def: def})
|
||||
continue
|
||||
}
|
||||
rest := letters[i+len(letter):]
|
||||
if strings.HasPrefix(rest, "=") {
|
||||
return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s",
|
||||
letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:])
|
||||
}
|
||||
return append(flags, flagArg{def, rest, rest != ""}), nil
|
||||
}
|
||||
return flags, nil
|
||||
}
|
||||
|
||||
func parseArgs(argv []string) (invocation, error) {
|
||||
in := invocation{repeat: 1, format: "text"}
|
||||
for i := 0; i < len(argv); i++ {
|
||||
arg := argv[i]
|
||||
if arg == "--" {
|
||||
in.args = append(in.args, argv[i+1:]...)
|
||||
return in, nil
|
||||
}
|
||||
if len(arg) < 2 || arg[0] != '-' {
|
||||
in.args = append(in.args, arg)
|
||||
continue
|
||||
}
|
||||
flags, err := splitFlags(arg)
|
||||
if err != nil {
|
||||
return in, err
|
||||
}
|
||||
for _, fl := range flags {
|
||||
if !fl.def.value {
|
||||
if fl.inline {
|
||||
return in, fmt.Errorf("--%s takes no value", fl.def.long)
|
||||
}
|
||||
_ = fl.def.set(&in, "")
|
||||
continue
|
||||
}
|
||||
value := fl.value
|
||||
if !fl.inline {
|
||||
if i++; i >= len(argv) {
|
||||
return in, fmt.Errorf("--%s needs a value", fl.def.long)
|
||||
}
|
||||
value = argv[i]
|
||||
}
|
||||
if err := fl.def.set(&in, value); err != nil {
|
||||
return in, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return in, nil
|
||||
}
|
||||
|
||||
// recordFormat is one way to write a record out: the line each record renders,
|
||||
// the header that precedes the first one, and the open/close frame plus the
|
||||
// between-record separator a document form needs.
|
||||
type recordFormat struct {
|
||||
header func(*fejkdata.Record) string
|
||||
line func(r *fejkdata.Record, table string) string
|
||||
open string
|
||||
close string
|
||||
sep string
|
||||
writes string // what it writes between values, which --separator would seem to set
|
||||
}
|
||||
|
||||
// recordFormats is every --format that writes records. json frames the records
|
||||
// as one array document; ndjson writes one object per line.
|
||||
var recordFormats = map[string]recordFormat{
|
||||
"csv": {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }, sep: "\n", writes: `always separates fields with ","`},
|
||||
"json": {line: jsonLine, open: "[", close: "]", sep: ",\n", writes: "writes the records as one JSON array"},
|
||||
"ndjson": {line: jsonLine, sep: "\n", writes: "writes one record per line"},
|
||||
"sql": {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }, sep: "\n", writes: "writes one INSERT per line"},
|
||||
}
|
||||
|
||||
func jsonLine(r *fejkdata.Record, _ string) string { return r.JSON() }
|
||||
|
||||
func (in invocation) writesRecords() bool {
|
||||
return in.format != "text"
|
||||
}
|
||||
|
||||
// formatNames lists the --format values, text included, for the misuse error.
|
||||
func formatNames() string {
|
||||
names := []string{"text"}
|
||||
for name := range recordFormats {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return strings.Join(names[:len(names)-1], ", ") + " or " + names[len(names)-1]
|
||||
}
|
||||
|
||||
// checkFlags rejects a flag value or combination that cannot run.
|
||||
func (in invocation) checkFlags() error {
|
||||
if _, ok := recordFormats[in.format]; !ok && in.format != "text" {
|
||||
return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format)
|
||||
}
|
||||
if in.tableSet && in.table == "" && in.format == "sql" {
|
||||
return errors.New("--table names the INSERT target, so it cannot be empty")
|
||||
}
|
||||
if f, records := recordFormats[in.format]; records && in.separatorSet && !in.list {
|
||||
return fmt.Errorf("--separator joins text renders; --format %s %s", in.format, f.writes)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func pipeHint(name string) string { return "echo 'id: {int(1,999)}' | " + name }
|
||||
|
||||
// operandError refuses an argument that is no flag, naming the spelling that reads what
|
||||
// it most likely means: dropping it for -, a file for a file's name, else a template
|
||||
// piped in.
|
||||
func operandError(arg, name string) error {
|
||||
unexpected := "unexpected argument '" + arg + "'"
|
||||
if arg == "-" {
|
||||
return fmt.Errorf("%s; stdin is read already, so drop it", unexpected)
|
||||
}
|
||||
if fi, err := os.Stat(arg); err == nil && fi.Mode().IsRegular() {
|
||||
return fmt.Errorf("%s; read a file from stdin: %s < %s", unexpected, name, shellQuoted(arg))
|
||||
}
|
||||
ref := "{/" + strings.TrimPrefix(arg, "/") + "}"
|
||||
_, argLone := loneReference(arg)
|
||||
inline, err := fejkdata.IsTemplate(arg)
|
||||
_, refLone := loneReference(ref)
|
||||
example := pipeHint(name)
|
||||
switch {
|
||||
case argLone || inline:
|
||||
example = piped(arg, name)
|
||||
case err == nil && refLone:
|
||||
example = piped(ref, name)
|
||||
}
|
||||
return fmt.Errorf("%s; the template is read from stdin: %s", unexpected, example)
|
||||
}
|
||||
|
||||
// check rejects a flag combination or an operand that cannot run, before any data is
|
||||
// read.
|
||||
func (in invocation) check(name string) error {
|
||||
if err := in.checkFlags(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(in.args) > 0 {
|
||||
return operandError(in.args[0], name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (in invocation) options(modules []fs.FS, systemSeed uint64) []fejkdata.Option {
|
||||
if in.noShipped {
|
||||
modules = nil
|
||||
}
|
||||
opts := []fejkdata.Option{fejkdata.WithDataFS(modules...)}
|
||||
for _, dir := range in.dirs {
|
||||
opts = append(opts, fejkdata.WithDataPath(dir))
|
||||
}
|
||||
seed := in.seed
|
||||
if !in.seeded {
|
||||
seed = systemSeed
|
||||
}
|
||||
return append(opts, fejkdata.WithSeed(seed))
|
||||
}
|
||||
|
||||
// write streams the input's renders to w, repeat of them joined by the separator; a write
|
||||
// failure surfaces from Flush, bufio keeping the first one.
|
||||
func (in invocation) write(f *fejkdata.Generator, src input, w io.Writer) error {
|
||||
if in.writesRecords() {
|
||||
return in.writeRecords(f, src, w)
|
||||
}
|
||||
t, err := f.NewTemplate(src.template)
|
||||
if err != nil {
|
||||
return templateError(err)
|
||||
}
|
||||
out := bufio.NewWriter(w)
|
||||
for i := 0; i < in.repeat; i++ {
|
||||
if i > 0 {
|
||||
out.WriteString(in.separator)
|
||||
}
|
||||
out.WriteString(t.Fake())
|
||||
}
|
||||
return out.Flush()
|
||||
}
|
||||
|
||||
// recordStream builds the record drawer for the input, plus the INSERT table
|
||||
// a sql format names.
|
||||
func (in invocation) recordStream(f *fejkdata.Generator, src input) (func() (*fejkdata.Record, error), string, error) {
|
||||
record := func() (*fejkdata.Record, error) { return f.FakeRecord(src.record) }
|
||||
if src.record == "" {
|
||||
t, err := f.NewRecordTemplate(src.template)
|
||||
if err != nil {
|
||||
return nil, "", templateError(spacedReference(src.template, err))
|
||||
}
|
||||
record = func() (*fejkdata.Record, error) { return t.Fake(), nil }
|
||||
}
|
||||
table := in.table
|
||||
if table == "" {
|
||||
table = defaultTable(src.record)
|
||||
}
|
||||
return record, table, nil
|
||||
}
|
||||
|
||||
// writeRecords streams a record per line in the chosen format, framing a document
|
||||
// form with its open/close brackets and a header preceding the first record.
|
||||
func (in invocation) writeRecords(f *fejkdata.Generator, src input, w io.Writer) error {
|
||||
record, table, err := in.recordStream(f, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
format := recordFormats[in.format]
|
||||
if in.repeat == 0 {
|
||||
return writeNoRecord(record, format, w)
|
||||
}
|
||||
out := bufio.NewWriter(w)
|
||||
if format.open != "" {
|
||||
out.WriteString(format.open)
|
||||
out.WriteByte('\n')
|
||||
}
|
||||
for i := 0; i < in.repeat; i++ {
|
||||
r, err := record()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if i == 0 && format.header != nil {
|
||||
out.WriteString(format.header(r))
|
||||
out.WriteByte('\n')
|
||||
}
|
||||
if i > 0 {
|
||||
out.WriteString(format.sep)
|
||||
}
|
||||
out.WriteString(format.line(r, table))
|
||||
}
|
||||
if format.close != "" {
|
||||
out.WriteByte('\n')
|
||||
out.WriteString(format.close)
|
||||
}
|
||||
out.WriteByte('\n')
|
||||
return out.Flush()
|
||||
}
|
||||
|
||||
// writeNoRecord writes zero records: the header where the format writes one, an empty document
|
||||
// where it frames one, else nothing. It draws one record, which names the header's columns, so a
|
||||
// lone reference to nothing fails as it would with records.
|
||||
func writeNoRecord(record func() (*fejkdata.Record, error), format recordFormat, w io.Writer) error {
|
||||
r, err := record()
|
||||
switch {
|
||||
case err != nil:
|
||||
return err
|
||||
case format.header != nil:
|
||||
_, err = io.WriteString(w, format.header(r)+"\n")
|
||||
case format.open != "":
|
||||
_, err = io.WriteString(w, format.open+format.close+"\n")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// defaultTable names the INSERT target when --table is absent: the path's last
|
||||
// segment, or "records" when stdin is not a lone reference.
|
||||
func defaultTable(path string) string {
|
||||
if path == "" {
|
||||
return "records"
|
||||
}
|
||||
segments := strings.Split(withoutSelectors(path), ".")
|
||||
return segments[len(segments)-1]
|
||||
}
|
||||
|
||||
// withoutSelectors is path with its [selectors] cut out.
|
||||
func withoutSelectors(path string) string {
|
||||
var names strings.Builder
|
||||
for depth, i := 0, 0; i < len(path); i++ {
|
||||
switch {
|
||||
case path[i] == '[':
|
||||
depth++
|
||||
case path[i] == ']':
|
||||
depth--
|
||||
case depth == 0:
|
||||
names.WriteByte(path[i])
|
||||
}
|
||||
}
|
||||
return names.String()
|
||||
}
|
||||
|
||||
// inputError marks a failure that is the input's own fault — nothing piped in, a template
|
||||
// that does not compile, or a stdin that cannot be read. run reports it as misuse (exit 2, with a
|
||||
// pointer to --help), unlike a lone reference to nothing under --format, which is a runtime
|
||||
// error (exit 1).
|
||||
type inputError struct{ error }
|
||||
|
||||
func (e inputError) Unwrap() error { return e.error }
|
||||
|
||||
// jsonSpace is the whitespace JSON allows around a value.
|
||||
const jsonSpace = " \t\r\n"
|
||||
|
||||
// input is what stdin holds: the template, and the path a lone reference names as a
|
||||
// record.
|
||||
type input struct {
|
||||
template string
|
||||
record string
|
||||
}
|
||||
|
||||
// parseInput keeps stdin whole as the template, and names a record where stdin is a lone
|
||||
// reference once JSON's surrounding whitespace, or one final newline, is cut.
|
||||
func parseInput(raw string) input {
|
||||
trimmed := strings.Trim(raw, jsonSpace)
|
||||
text := raw
|
||||
if json.Valid([]byte(trimmed)) {
|
||||
text = trimmed
|
||||
}
|
||||
text, found := strings.CutSuffix(text, "\n")
|
||||
if found {
|
||||
text = strings.TrimSuffix(text, "\r")
|
||||
}
|
||||
record, _ := loneReference(text)
|
||||
return input{template: raw, record: record}
|
||||
}
|
||||
|
||||
// loneReference returns the path that text reads when it is one reference from the root
|
||||
// and nothing else, bare or as a JSON string.
|
||||
func loneReference(text string) (string, bool) {
|
||||
var unquoted string
|
||||
if strings.HasPrefix(text, `"`) && strings.HasSuffix(text, `"`) && json.Unmarshal([]byte(text), &unquoted) == nil {
|
||||
text = unquoted
|
||||
}
|
||||
path, opens := strings.CutPrefix(text, "{/")
|
||||
path, closes := strings.CutSuffix(path, "}")
|
||||
if !opens || !closes || !isPath(path) {
|
||||
return "", false
|
||||
}
|
||||
return path, true
|
||||
}
|
||||
|
||||
// isPath reports whether a reference's body after its / is a path alone: no second /, no
|
||||
// folder sigil, no empty segment, and none of the |, ( or " as " a token reads outside a selector.
|
||||
// It copies the grammar, since the CLI reads only the library's public API.
|
||||
func isPath(body string) bool {
|
||||
if body == "" || strings.ContainsAny(body[:1], "/.") {
|
||||
return false
|
||||
}
|
||||
names := withoutSelectors(body)
|
||||
if strings.ContainsAny(names, "|(") || strings.Contains(names, " as ") || hasEmptySegment(names) {
|
||||
return false
|
||||
}
|
||||
inline, err := fejkdata.IsTemplate(body)
|
||||
return !inline && err == nil
|
||||
}
|
||||
|
||||
// hasEmptySegment reports whether a dotted path holds a segment with no name: two dots are a
|
||||
// step up, and only a step up may end the path.
|
||||
func hasEmptySegment(path string) bool {
|
||||
segments := strings.Split(path, ".")
|
||||
stepUp := false
|
||||
for i, seg := range segments[1:] {
|
||||
last := i == len(segments)-2
|
||||
switch {
|
||||
case seg != "":
|
||||
stepUp = false
|
||||
case stepUp != last:
|
||||
return true
|
||||
default:
|
||||
stepUp = !stepUp
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// spacedReference adds to a record refusal that whitespace around a lone reference made
|
||||
// the template text.
|
||||
func spacedReference(template string, err error) error {
|
||||
if trimmed := strings.TrimSpace(template); trimmed != template && errors.Is(err, fejkdata.ErrNoColumns) {
|
||||
if path, lone := loneReference(trimmed); lone {
|
||||
return fmt.Errorf("%w; the whitespace around {/%s} makes it text, so remove it to write the record %s", err, path, path)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// piped is the command piping template into the command called name; some shells' echo
|
||||
// reads a backslash as an escape, so a template holding one goes through printf.
|
||||
func piped(template, name string) string {
|
||||
if strings.ContainsRune(template, '\\') {
|
||||
return "printf '%s' " + shellQuoted(template) + " | " + name
|
||||
}
|
||||
return "echo " + shellQuoted(template) + " | " + name
|
||||
}
|
||||
|
||||
// shellQuoted is s as a shell reads it back: bare where it holds nothing a shell expands.
|
||||
func shellQuoted(s string) string {
|
||||
if s != "" && !strings.ContainsAny(s, " \t\r\n'\"$`\\*?[]{}()<>|&;#~!") {
|
||||
return s
|
||||
}
|
||||
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
|
||||
}
|
||||
|
||||
// Main runs the command on the process's arguments and standard streams over modules, each
|
||||
// passed to [fejkdata.WithDataFS], and returns the exit code: 0 ok, 1 runtime error,
|
||||
// 2 misuse. --no-shipped-data leaves modules out.
|
||||
func Main(modules ...fs.FS) int {
|
||||
var stdin io.Reader = os.Stdin
|
||||
if fi, err := os.Stdin.Stat(); err == nil && fi.Mode()&os.ModeCharDevice != 0 {
|
||||
stdin = nil
|
||||
}
|
||||
return run(commandName(os.Args[0]), modules, os.Args[1:], rand.Uint64(), stdin, os.Stdout, os.Stderr)
|
||||
}
|
||||
|
||||
// commandName is the name the command runs under: the file arg0 names, less a Windows
|
||||
// .exe, or fejkdata where arg0 names none.
|
||||
func commandName(arg0 string) string {
|
||||
name := filepath.Base(arg0)
|
||||
if ext := filepath.Ext(name); strings.EqualFold(ext, ".exe") {
|
||||
name = strings.TrimSuffix(name, ext)
|
||||
}
|
||||
if name == "." || name == "" {
|
||||
return "fejkdata"
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// run is Main over the given streams, run as the command name, seeded by systemSeed where
|
||||
// no --seed is given. A nil stdin is a terminal, which holds no template.
|
||||
func run(name string, modules []fs.FS, args []string, systemSeed uint64, stdin io.Reader, stdout, stderr io.Writer) int {
|
||||
in, err := parseArgs(args)
|
||||
if err != nil {
|
||||
return misuse(stderr, name, err)
|
||||
}
|
||||
if in.help {
|
||||
io.WriteString(stdout, strings.ReplaceAll(usage, "COMMAND", name))
|
||||
return 0
|
||||
}
|
||||
if in.version {
|
||||
info, _ := debug.ReadBuildInfo()
|
||||
fmt.Fprintln(stdout, versionLine(name, info))
|
||||
return 0
|
||||
}
|
||||
if err := in.check(name); err != nil {
|
||||
return misuse(stderr, name, err)
|
||||
}
|
||||
var src input
|
||||
if !in.list {
|
||||
if src, err = readStdin(stdin, name); err != nil {
|
||||
return fail(stderr, name, err)
|
||||
}
|
||||
}
|
||||
f, err := fejkdata.New(in.options(modules, systemSeed)...)
|
||||
if err != nil {
|
||||
return fail(stderr, name, err)
|
||||
}
|
||||
if in.list {
|
||||
for _, p := range f.List() {
|
||||
fmt.Fprintln(stdout, p)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if err := in.write(f, src, stdout); err != nil {
|
||||
return fail(stderr, name, err)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func readStdin(stdin io.Reader, name string) (input, error) {
|
||||
if stdin == nil {
|
||||
return input{}, inputError{fmt.Errorf("the template comes from stdin, and nothing is piped in: %s; %s --list prints the paths a {/path} reference reads", pipeHint(name), name)}
|
||||
}
|
||||
raw, err := io.ReadAll(stdin)
|
||||
if err != nil {
|
||||
return input{}, inputError{fmt.Errorf("stdin cannot be read: %w", err)}
|
||||
}
|
||||
return parseInput(string(raw)), nil
|
||||
}
|
||||
|
||||
// templateError is misuse unless the data the template reads failed to load.
|
||||
func templateError(err error) error {
|
||||
if errors.Is(err, fejkdata.ErrLoad) {
|
||||
return err
|
||||
}
|
||||
return inputError{err}
|
||||
}
|
||||
|
||||
func fail(stderr io.Writer, name string, err error) int {
|
||||
var te inputError
|
||||
if errors.As(err, &te) {
|
||||
return misuse(stderr, name, te.error)
|
||||
}
|
||||
report(stderr, name, err)
|
||||
if errors.Is(err, fejkdata.ErrNoColumns) {
|
||||
fmt.Fprintln(stderr, "wrap that JSON in single quotes, which keep a shell from expanding its braces")
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
func misuse(stderr io.Writer, name string, err error) int {
|
||||
report(stderr, name, err)
|
||||
fmt.Fprintf(stderr, "try '%s --help'\n", name)
|
||||
return 2
|
||||
}
|
||||
|
||||
// report writes err under the command name, in place of the library's own on each line.
|
||||
func report(stderr io.Writer, name string, err error) {
|
||||
msg := strings.ReplaceAll(strings.TrimPrefix(err.Error(), "fejkdata: "), "\nfejkdata: ", "\n"+name+": ")
|
||||
fmt.Fprintf(stderr, "%s: %s\n", name, msg)
|
||||
}
|
||||
|
||||
// fejkdataModule is the path of the module holding the library; the root package's path is
|
||||
// the module's.
|
||||
var fejkdataModule = reflect.TypeOf((*fejkdata.Generator)(nil)).Elem().PkgPath()
|
||||
|
||||
// versionLine names the fejkdata version the binary is built with. Where another module's
|
||||
// main package carries the CLI, the main module's version comes first, under the command name.
|
||||
func versionLine(name string, info *debug.BuildInfo) string {
|
||||
if info == nil {
|
||||
return "fejkdata devel"
|
||||
}
|
||||
if info.Main.Path == fejkdataModule {
|
||||
return "fejkdata " + moduleVersion(info.Main)
|
||||
}
|
||||
for _, m := range info.Deps {
|
||||
if m.Path == fejkdataModule {
|
||||
return name + " " + moduleVersion(info.Main) + ", fejkdata " + moduleVersion(*m)
|
||||
}
|
||||
}
|
||||
return name + " " + moduleVersion(info.Main) + ", fejkdata devel"
|
||||
}
|
||||
|
||||
// moduleVersion is the version m is built at: the replacement's where a replace directive
|
||||
// names one, and "devel" where it carries none, as a local directory does.
|
||||
func moduleVersion(m debug.Module) string {
|
||||
if m.Replace != nil {
|
||||
m = *m.Replace
|
||||
}
|
||||
if m.Version == "" || m.Version == "(devel)" {
|
||||
return "devel"
|
||||
}
|
||||
return m.Version
|
||||
}
|
||||
-961
@@ -1,961 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
)
|
||||
|
||||
// shipped holds the shipped modules as read from the checkout, each folder under ../data
|
||||
// holding a manifest, so these tests link no data package.
|
||||
var shipped = func() []fs.FS {
|
||||
var modules []fs.FS
|
||||
err := filepath.WalkDir(filepath.Join("..", "data"), func(path string, d fs.DirEntry, err error) error {
|
||||
if err == nil && d.Name() == ".fejkdata.json" {
|
||||
modules = append(modules, os.DirFS(filepath.Dir(path)))
|
||||
}
|
||||
return err
|
||||
})
|
||||
if err != nil || len(modules) == 0 {
|
||||
panic(fmt.Sprintf("no shipped module found under ../data: %v", err))
|
||||
}
|
||||
return modules
|
||||
}()
|
||||
|
||||
// svSE, enUS and misc are copies of a shipped module's files without its manifest, so a test
|
||||
// layers its categories unprefixed with -d.
|
||||
var svSE, enUS, misc string
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
tmp, err := os.MkdirTemp("", "fejkdata-cli-test")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
svSE, enUS, misc = bareCopy(tmp, "sv_SE"), bareCopy(tmp, "en_US"), bareCopy(tmp, "misc")
|
||||
code := m.Run()
|
||||
os.RemoveAll(tmp)
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
// bareCopy copies the category and rows files of the shipped module in data/<module> into a
|
||||
// folder under tmp, and returns that folder.
|
||||
func bareCopy(tmp, module string) string {
|
||||
src, dst := filepath.Join("..", "data", module), filepath.Join(tmp, module)
|
||||
if err := os.Mkdir(dst, 0o755); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for _, pattern := range []string{"*.json", "*.tsv"} {
|
||||
files, err := filepath.Glob(filepath.Join(src, pattern))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
if strings.HasPrefix(filepath.Base(f), ".") {
|
||||
continue
|
||||
}
|
||||
b, err := os.ReadFile(f)
|
||||
if err == nil {
|
||||
err = os.WriteFile(filepath.Join(dst, filepath.Base(f)), b, 0o644)
|
||||
}
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
func runOut(stdin string, args ...string) (int, string, string) {
|
||||
var out, errb bytes.Buffer
|
||||
code := run("fejkdata", shipped, args, 1, strings.NewReader(stdin), &out, &errb)
|
||||
return code, out.String(), errb.String()
|
||||
}
|
||||
|
||||
// runTerminal runs as from a shell with nothing piped in.
|
||||
func runTerminal(args ...string) (int, string, string) {
|
||||
var out, errb bytes.Buffer
|
||||
code := run("fejkdata", shipped, args, 1, nil, &out, &errb)
|
||||
return code, out.String(), errb.String()
|
||||
}
|
||||
|
||||
func TestRunLoadsTheModulesItIsGiven(t *testing.T) {
|
||||
module := fstest.MapFS{"pet.json": {Data: []byte(`{"format":"{name}","name":["Fido"]}`)}}
|
||||
var out, errb bytes.Buffer
|
||||
if code := run("fejkdata", []fs.FS{module}, nil, 1, strings.NewReader("{/pet}"), &out, &errb); code != 0 || out.String() != "Fido" {
|
||||
t.Errorf("run over a module holding pet = %d, %q, %q, want Fido", code, out.String(), errb.String())
|
||||
}
|
||||
out.Reset()
|
||||
if code := run("fejkdata", []fs.FS{module}, []string{"--list"}, 1, nil, &out, &errb); code != 0 || out.String() != "pet\npet.name\n" {
|
||||
t.Errorf("--list over a module holding pet = %d, %q, want pet and pet.name only", code, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOutputsValue(t *testing.T) {
|
||||
code, out, errb := runOut("{/person}", "--data-path", svSE)
|
||||
if code != 0 {
|
||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
if out == "" || strings.Contains(out, "\n") {
|
||||
t.Fatalf("output = %q, stderr=%q; want one value and nothing added", out, errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDotPath(t *testing.T) {
|
||||
code, full, _ := runOut("{/person}", "--seed", "7", "--data-path", svSE)
|
||||
if code != 0 {
|
||||
t.Fatalf("person run = %d", code)
|
||||
}
|
||||
code, last, errb := runOut("{/person.last}", "--seed", "7", "--data-path", svSE)
|
||||
if code != 0 {
|
||||
t.Fatalf("person.last run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
if strings.TrimSpace(last) == "" {
|
||||
t.Fatal("empty person.last output")
|
||||
}
|
||||
if last == full {
|
||||
t.Errorf("person.last %q should differ from person %q", last, full)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunWithoutASeedTakesTheSystemSeed(t *testing.T) {
|
||||
var drawn, seeded, errb bytes.Buffer
|
||||
if code := run("fejkdata", shipped, nil, 42, strings.NewReader("{uuid()}"), &drawn, &errb); code != 0 {
|
||||
t.Fatalf("run with system seed 42 = %d, stderr=%q", code, errb.String())
|
||||
}
|
||||
if code := run("fejkdata", shipped, []string{"--seed", "42"}, 7, strings.NewReader("{uuid()}"), &seeded, &errb); code != 0 {
|
||||
t.Fatalf("run --seed 42 = %d, stderr=%q", code, errb.String())
|
||||
}
|
||||
if drawn.String() != seeded.String() {
|
||||
t.Fatalf("run with system seed 42 = %q, run --seed 42 = %q; want the same draw", drawn.String(), seeded.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSeedSpellings(t *testing.T) {
|
||||
_, want, _ := runOut("{/address}", "--seed", "42", "--data-path", svSE)
|
||||
for _, args := range [][]string{
|
||||
{"--seed=42", "--data-path", svSE},
|
||||
{"-s", "42", "-d", svSE},
|
||||
{"--data-path", svSE, "--seed", "42"},
|
||||
} {
|
||||
code, got, errb := runOut("{/address}", args...)
|
||||
if code != 0 {
|
||||
t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("run(%v) = %q, want %q", args, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunShortFlagValues(t *testing.T) {
|
||||
_, want, _ := runOut("{/address}", "--seed", "42", "--data-path", svSE)
|
||||
for _, args := range [][]string{
|
||||
{"-s42", "-d", svSE},
|
||||
{"-s", "42", "-d" + svSE},
|
||||
{"-d", svSE, "-s42"},
|
||||
} {
|
||||
code, got, errb := runOut("{/address}", args...)
|
||||
if code != 0 {
|
||||
t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("run(%v) = %q, want %q", args, got, want)
|
||||
}
|
||||
}
|
||||
_, three, _ := runOut("{/word}\n", "-s", "1", "-n3", "-d", svSE)
|
||||
if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 {
|
||||
t.Errorf("-n3 gave %d lines: %q", len(lines), three)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{[]string{"-s=42", "-d", svSE}, "-s42"},
|
||||
{[]string{"-s=42", "-d", svSE}, "--seed=42"},
|
||||
{[]string{"-d=" + svSE}, "--data-path="},
|
||||
{[]string{"-nd", "3", "-d", svSE}, `--repeat needs an integer in 0..1048576, got "d"`},
|
||||
{[]string{"--seed=", "-d", svSE}, `--seed needs an unsigned integer, got ""`},
|
||||
} {
|
||||
code, out, errb := runOut("{/word}", c.args...)
|
||||
if code != 2 || out != "" {
|
||||
t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out)
|
||||
}
|
||||
if !strings.Contains(errb, c.want) {
|
||||
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
|
||||
}
|
||||
}
|
||||
code, out, _ := runOut("", "-hd", svSE)
|
||||
if code != 0 || !strings.Contains(out, "Usage") {
|
||||
t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDoubleDashEndsFlags(t *testing.T) {
|
||||
code, out, errb := runOut("{/person}", "--data-path", svSE, "--")
|
||||
if code != 0 || strings.TrimSpace(out) == "" {
|
||||
t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
|
||||
}
|
||||
code, _, errb = runOut("{/person}", "--data-path", svSE, "--", "--list")
|
||||
if code != 2 || !strings.Contains(errb, "stdin") {
|
||||
t.Errorf("after --, --list should be an argument: code %d, stderr %q", code, errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSingleDashLongIsRejected(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{[]string{"-seed", "42", "-d", svSE}, "use --seed"},
|
||||
{[]string{"-seed=42", "-d", svSE}, "use --seed"},
|
||||
{[]string{"-data-path", svSE}, "use --data-path"},
|
||||
{[]string{"-list", "-d", svSE}, "use --list"},
|
||||
{[]string{"--nope", "-d", svSE}, "unknown flag --nope"},
|
||||
{[]string{"-x", "-d", svSE}, "unknown flag -x"},
|
||||
} {
|
||||
code, _, errb := runOut("{/person}", c.args...)
|
||||
if code != 2 {
|
||||
t.Errorf("run(%v) = %d, want 2", c.args, code)
|
||||
}
|
||||
if !strings.Contains(errb, c.want) {
|
||||
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFlagValues(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{[]string{"--seed", "-d", svSE}, `--seed needs an unsigned integer, got "-d"`},
|
||||
{[]string{"-d", svSE, "--seed"}, "--seed needs a value"},
|
||||
{[]string{"--seed", "x", "-d", svSE}, "--seed"},
|
||||
{[]string{"--list=1", "-d", svSE}, "--list takes no value"},
|
||||
{[]string{"--repeat", "-1", "-d", svSE}, "--repeat"},
|
||||
{[]string{"-n", "-1", "-d", svSE}, "--repeat"},
|
||||
} {
|
||||
code, _, errb := runOut("{/person}", c.args...)
|
||||
if code != 2 {
|
||||
t.Errorf("run(%v) = %d, want 2", c.args, code)
|
||||
}
|
||||
if !strings.Contains(errb, c.want) {
|
||||
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRepeat(t *testing.T) {
|
||||
code, out, errb := runOut("{/word}\n", "--seed", "1", "--repeat", "3", "--data-path", svSE)
|
||||
if code != 0 {
|
||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
|
||||
if len(lines) != 3 {
|
||||
t.Errorf("repeat=3 gave %d lines: %q", len(lines), out)
|
||||
}
|
||||
_, short, _ := runOut("{/word}\n", "--seed", "1", "-n", "3", "-d", svSE)
|
||||
if short != out {
|
||||
t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSeparator(t *testing.T) {
|
||||
code, out, errb := runOut("{/word}", "--repeat", "3", "--separator", ",", "--data-path", svSE)
|
||||
if code != 0 {
|
||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
if n := strings.Count(out, "\n"); n != 0 {
|
||||
t.Errorf("want no newline, got %d: %q", n, out)
|
||||
}
|
||||
if !strings.Contains(out, ",") {
|
||||
t.Errorf("values should be comma-joined: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRepeatAdvancesRNG(t *testing.T) {
|
||||
code, out, errb := runOut("{/person}\n", "--seed", "1", "--repeat", "5", "--data-path", svSE)
|
||||
if code != 0 {
|
||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
uniq := map[string]bool{}
|
||||
for _, l := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
|
||||
uniq[l] = true
|
||||
}
|
||||
if len(uniq) < 2 {
|
||||
t.Errorf("repeat should vary output, all identical: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMisuse(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
stdin string
|
||||
args []string
|
||||
}{
|
||||
{"{/person}", []string{"-d", svSE, "word"}},
|
||||
{"", []string{"-d", svSE, "--list", "person"}},
|
||||
{"{/sv_SE.person}", []string{"--no-shipped-data"}},
|
||||
} {
|
||||
code, out, errb := runOut(c.stdin, c.args...)
|
||||
if code != 2 {
|
||||
t.Errorf("run(%q, %v) = %d, want 2", c.stdin, c.args, code)
|
||||
}
|
||||
if !strings.Contains(errb, "try 'fejkdata --help'") {
|
||||
t.Errorf("run(%q, %v) stderr = %q, want a pointer to --help", c.stdin, c.args, errb)
|
||||
}
|
||||
if out != "" {
|
||||
t.Errorf("run(%q, %v) stdout = %q, want nothing", c.stdin, c.args, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunTakesNoOperand(t *testing.T) {
|
||||
file := filepath.Join(t.TempDir(), "users.tmpl")
|
||||
if err := os.WriteFile(file, []byte("{/sv_SE.person}"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{[]string{"sv_SE.person"}, "echo '{/sv_SE.person}' | fejkdata"},
|
||||
{[]string{"/en_US.address"}, "echo '{/en_US.address}' | fejkdata"},
|
||||
{[]string{"{/en_US.person}"}, "echo '{/en_US.person}' | fejkdata"},
|
||||
{[]string{"name: {/sv_SE.person.last}"}, "echo 'name: {/sv_SE.person.last}' | fejkdata"},
|
||||
{[]string{"geo.US.locality[O'Fallon].name"}, `echo '{/geo.US.locality[O'\''Fallon].name}' | fejkdata`},
|
||||
{[]string{"en_US.address"}, "echo '{/en_US.address}' | fejkdata"},
|
||||
{[]string{`C:\temp {/sv_SE.person}`}, `printf '%s' 'C:\temp {/sv_SE.person}' | fejkdata`},
|
||||
{[]string{"-"}, "drop it"},
|
||||
{[]string{file}, "fejkdata < " + file},
|
||||
} {
|
||||
code, out, errb := runOut("", c.args...)
|
||||
if code != 2 || out != "" || !strings.Contains(errb, "unexpected argument") || !strings.Contains(errb, c.want) {
|
||||
t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type failingReader struct{}
|
||||
|
||||
func (failingReader) Read([]byte) (int, error) { return 0, errors.New("bad file descriptor") }
|
||||
|
||||
func TestRunUnreadableStdinIsMisuse(t *testing.T) {
|
||||
var out, errb bytes.Buffer
|
||||
if code := run("fejkdata", shipped, nil, 1, failingReader{}, &out, &errb); code != 2 || !strings.Contains(errb.String(), "cannot be read") {
|
||||
t.Errorf("run with an unreadable stdin = %d, %q; want misuse naming it", code, errb.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunTerminalStdinIsMisuse(t *testing.T) {
|
||||
code, out, errb := runTerminal("--seed", "1")
|
||||
if code != 2 || out != "" || !strings.Contains(errb, "nothing is piped in: echo 'id: {int(1,999)}' | fejkdata; fejkdata --list prints the paths a {/path} reference reads\n") {
|
||||
t.Errorf("run with nothing piped = %d, %q, %q; want misuse showing a template piped to fejkdata", code, out, errb)
|
||||
}
|
||||
for _, args := range [][]string{{"--list"}, {"--help"}, {"--version"}} {
|
||||
if code, out, errb := runTerminal(args...); code != 0 || out == "" {
|
||||
t.Errorf("run(%v) with nothing piped = %d, %q, %q; want it to run, reading no stdin", args, code, out, errb)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRendersExactlyWhatIsSent(t *testing.T) {
|
||||
for stdin, want := range map[string]string{
|
||||
"hihi": "hihi",
|
||||
"hihi\n": "hihi\n",
|
||||
"hihi\r\n": "hihi\r\n",
|
||||
"hihi\n\n": "hihi\n\n",
|
||||
"{{x}}\n": "{x}\n",
|
||||
"\"a\"\n": "a",
|
||||
" \t\"a\"\r\n\n": "a",
|
||||
"{/misc.territory[SE].alpha2}": "SE",
|
||||
"{/misc.territory[SE].alpha2}\n": "SE\n",
|
||||
} {
|
||||
code, out, errb := runOut(stdin, "--seed", "1")
|
||||
if code != 0 || out != want {
|
||||
t.Errorf("run(%q) = %d, %q, %q; want %q", stdin, code, out, errb, want)
|
||||
}
|
||||
}
|
||||
if code, out, errb := runOut("{\"format\":\"{x}\\n\",\"x\":[\"a\",\"b\"]}\n"); code != 0 || (out != "a\n" && out != "b\n") {
|
||||
t.Errorf("a JSON template's format = %d, %q, %q; want its own newline only", code, out, errb)
|
||||
}
|
||||
if code, out, errb := runOut("{/sv_SE.word}\n", "--seed", "1", "-n", "3", "--separator", ","); code != 0 || strings.Count(out, "\n,") != 2 || !strings.HasSuffix(out, "\n") {
|
||||
t.Errorf("echo's newline and a separator = %d, %q, %q; want each render's newline, then the separator", code, out, errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRendersBlankStdin(t *testing.T) {
|
||||
for _, stdin := range []string{"", "\n", " \r\n\t"} {
|
||||
if code, out, errb := runOut(stdin); code != 0 || out != stdin {
|
||||
t.Errorf("run(%q) = %d, %q, %q; want the blank template rendered as it is", stdin, code, out, errb)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunTextPrintsAsWritten(t *testing.T) {
|
||||
for _, stdin := range []string{"sv_SE.person", "[Skåne län]", "x[1]y", `say "hi"`} {
|
||||
code, out, errb := runOut(stdin)
|
||||
if code != 0 || out != stdin {
|
||||
t.Errorf("run(%q) = %d, %q, %q; want the text as written", stdin, code, out, errb)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMultipleDataPaths(t *testing.T) {
|
||||
code, out, errb := runOut("{/person}", "-d", enUS, "--data-path", svSE)
|
||||
if code != 0 {
|
||||
t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb)
|
||||
}
|
||||
if strings.TrimSpace(out) == "" {
|
||||
t.Fatal("empty output for multi-dir run")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUnknownCategoryFails(t *testing.T) {
|
||||
code, _, errb := runOut("{/nope}", "--data-path", svSE, "--format", "csv")
|
||||
if code != 1 {
|
||||
t.Fatalf("run = %d, want 1", code)
|
||||
}
|
||||
if !strings.Contains(errb, "nope") {
|
||||
t.Errorf("stderr %q should name the unknown category", errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunList(t *testing.T) {
|
||||
code, out, errb := runOut("", "--data-path", svSE, "--list")
|
||||
if code != 0 {
|
||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
for _, want := range []string{"person", "person.last", "address"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("list output missing %q:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunHelpOnStdout(t *testing.T) {
|
||||
for _, h := range []string{"-h", "--help"} {
|
||||
code, out, errb := runOut("", h)
|
||||
if code != 0 {
|
||||
t.Errorf("%s = %d, want 0", h, code)
|
||||
}
|
||||
if !strings.Contains(out, "Usage") || errb != "" {
|
||||
t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionLineNamesFejkdatasVersion(t *testing.T) {
|
||||
fejkdata := func(version string, replace *debug.Module) *debug.Module {
|
||||
return &debug.Module{Path: "github.com/larvit/fejkdata", Version: version, Replace: replace}
|
||||
}
|
||||
mycli := debug.Module{Path: "example.com/mycli", Version: "v9.0.0"}
|
||||
for _, c := range []struct {
|
||||
info *debug.BuildInfo
|
||||
want string
|
||||
}{
|
||||
{&debug.BuildInfo{Main: *fejkdata("v0.1.0", nil)}, "fejkdata v0.1.0"},
|
||||
{&debug.BuildInfo{Main: *fejkdata("(devel)", nil)}, "fejkdata devel"},
|
||||
{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", nil)}}, "mycli v9.0.0, fejkdata v0.1.0"},
|
||||
{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", &debug.Module{Path: "../fejkdata"})}}, "mycli v9.0.0, fejkdata devel"},
|
||||
{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", &debug.Module{Path: "example.com/fork", Version: "v0.1.1"})}}, "mycli v9.0.0, fejkdata v0.1.1"},
|
||||
{&debug.BuildInfo{Main: mycli}, "mycli v9.0.0, fejkdata devel"},
|
||||
{nil, "fejkdata devel"},
|
||||
} {
|
||||
if got := versionLine("mycli", c.info); got != c.want {
|
||||
t.Errorf("versionLine(mycli, %+v) = %q, want %q", c.info, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommandNameIsTheFileItRunsFrom(t *testing.T) {
|
||||
for arg0, want := range map[string]string{
|
||||
"/usr/local/bin/mycli": "mycli",
|
||||
"mycli.EXE": "mycli",
|
||||
"mycli.exe": "mycli",
|
||||
"my.cli": "my.cli",
|
||||
"": "fejkdata",
|
||||
} {
|
||||
if got := commandName(arg0); got != want {
|
||||
t.Errorf("commandName(%q) = %q, want %q", arg0, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNamesItselfByTheCommandName(t *testing.T) {
|
||||
var out, errb bytes.Buffer
|
||||
if code := run("mycli", shipped, []string{"--nope"}, 1, nil, &out, &errb); code != 2 || errb.String() != "mycli: unknown flag --nope\ntry 'mycli --help'\n" {
|
||||
t.Errorf("mycli --nope = %d, %q", code, errb.String())
|
||||
}
|
||||
errb.Reset()
|
||||
if code := run("mycli", shipped, nil, 1, nil, &out, &errb); code != 2 || !strings.Contains(errb.String(), "| mycli; mycli --list prints") {
|
||||
t.Errorf("mycli with nothing piped = %d, %q, want a pipe into mycli", code, errb.String())
|
||||
}
|
||||
errb.Reset()
|
||||
if code := run("mycli", []fs.FS{nil, nil}, nil, 1, strings.NewReader("x"), &out, &errb); code != 1 || strings.Count(errb.String(), "mycli: ") != 2 || strings.Contains(errb.String(), "fejkdata") {
|
||||
t.Errorf("mycli over two nil modules = %d, %q, want each line under mycli", code, errb.String())
|
||||
}
|
||||
errb.Reset()
|
||||
if code := run("mycli", shipped, []string{"-d", "/nope"}, 1, strings.NewReader("{/sv_SE.person}"), &out, &errb); code != 1 || !strings.HasPrefix(errb.String(), "mycli: ") || strings.Contains(errb.String(), "fejkdata") {
|
||||
t.Errorf("mycli -d /nope = %d, %q, want a load error under mycli", code, errb.String())
|
||||
}
|
||||
if code := run("mycli", shipped, []string{"--help"}, 1, nil, &out, &errb); code != 0 || !strings.HasPrefix(out.String(), "Usage: <template> | mycli [flags]\n") || strings.Contains(out.String(), "fejkdata") {
|
||||
t.Errorf("mycli --help = %d, %q, want mycli named throughout", code, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunVersion(t *testing.T) {
|
||||
code, out, errb := runOut("", "--version")
|
||||
if code != 0 {
|
||||
t.Fatalf("--version = %d, stderr=%q", code, errb)
|
||||
}
|
||||
version, ok := strings.CutPrefix(out, "fejkdata ")
|
||||
if !ok || strings.TrimSpace(version) == "" {
|
||||
t.Errorf("--version = %q, want the command name and a version on stdout", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMissingDirFails(t *testing.T) {
|
||||
code, _, errb := runOut("{/person}", "--data-path", "../data/nope")
|
||||
if code != 1 {
|
||||
t.Fatalf("run = %d, want 1", code)
|
||||
}
|
||||
if errb == "" {
|
||||
t.Error("want an error message on stderr")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunShippedDataByDefault(t *testing.T) {
|
||||
code, out, errb := runOut("{/sv_SE.person}", "--seed", "1")
|
||||
if code != 0 || strings.TrimSpace(out) == "" {
|
||||
t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
|
||||
}
|
||||
code, list, _ := runOut("", "--list")
|
||||
if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") {
|
||||
t.Errorf("--list without --data-path = %d, %q", code, list)
|
||||
}
|
||||
code, _, errb = runOut("{/person}")
|
||||
if code != 2 || !strings.Contains(errb, "person") {
|
||||
t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSelectsATableRow(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for name, content := range map[string]string{
|
||||
"region.json": `{"format":"{name}","rows":"region.tsv","key":"code","name":"name"}`,
|
||||
"region.tsv": "code\tname\n01\tStockholms län\n12\tSkåne län\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if code, out, errb := runOut("{/region[12]}", "--no-shipped-data", "-d", dir); code != 0 || out != "Skåne län" {
|
||||
t.Fatalf("{/region[12]} = %d, %q, stderr=%q", code, out, errb)
|
||||
}
|
||||
if code, out, _ := runOut("{/region[Skåne län].code}", "--no-shipped-data", "-d", dir); code != 0 || out != "12" {
|
||||
t.Fatalf("{/region[Skåne län].code} = %d, %q", code, out)
|
||||
}
|
||||
if code, out, _ := runOut("{/region[12]}", "--no-shipped-data", "-d", dir, "--format", "sql"); code != 0 || out != `INSERT INTO "region" ("code", "name") VALUES ('12', 'Skåne län');`+"\n" {
|
||||
t.Fatalf("--format sql {/region[12]} = %d, %q, want the table named without its selector", code, out)
|
||||
}
|
||||
if code, _, errb := runOut("{/region[99]}", "--no-shipped-data", "-d", dir); code != 2 || !strings.Contains(errb, `"99"`) {
|
||||
t.Fatalf("{/region[99]} = %d, stderr=%q, want misuse naming the row", code, errb)
|
||||
}
|
||||
if code, out, _ := runOut("{/misc.territory[SE]}", "--seed", "1", "--format", "csv"); code != 0 || !strings.HasPrefix(out, "alpha2,") || !strings.Contains(out, "\nSE,SWE,") {
|
||||
t.Fatalf("--format csv {/misc.territory[SE]} = %d, %q", code, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUsageExamplesRenderWithNoData(t *testing.T) {
|
||||
examples := regexp.MustCompile(`(?m)^ echo '([^']*)' \| COMMAND$`).FindAllStringSubmatch(usage+"\n "+pipeHint("COMMAND")+"\n", -1)
|
||||
if len(examples) < 3 {
|
||||
t.Fatalf("usage and pipe hint hold %d echo examples, want the help's two and the hint's", len(examples))
|
||||
}
|
||||
for _, e := range examples {
|
||||
if code, out, errb := runOut(e[1], "--no-shipped-data"); code != 0 || strings.TrimSpace(out) == "" {
|
||||
t.Errorf("echo %q with no data: run = %d, %q, stderr %q", e[1], code, out, errb)
|
||||
}
|
||||
}
|
||||
for _, token := range regexp.MustCompile(`\{/[^}]*\}`).FindAllString(usage, -1) {
|
||||
if token != "{/path}" && token != "{/table[KEY]}" {
|
||||
t.Errorf("usage names %s, a path a custom CLI may not carry", token)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(usage, "\nExit codes: 0 success, 1 runtime error, 2 misuse.\n") {
|
||||
t.Errorf("usage does not list the exit codes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOperandMisuseBeforeLoad(t *testing.T) {
|
||||
code, _, errb := runOut("", "--no-shipped-data", "x")
|
||||
if code != 2 || !strings.Contains(errb, "unexpected argument") || strings.Contains(errb, "--data-path") {
|
||||
t.Fatalf("an operand with no data = %d, %q; want the operand refused before any load", code, errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunInlineTemplate(t *testing.T) {
|
||||
code, out, errb := runOut("name: {/sv_SE.person.last}", "--seed", "1")
|
||||
if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
|
||||
t.Fatalf("inline format string = %d, %q, stderr %q", code, out, errb)
|
||||
}
|
||||
code, out, errb = runOut(`{"format":"name: {x}","x":["bosse","lina"]}`, "--seed", "1")
|
||||
if code != 0 || (out != "name: bosse" && out != "name: lina") {
|
||||
t.Fatalf("inline JSON template = %d, %q, want one name, stderr %q", code, out, errb)
|
||||
}
|
||||
code, out, errb = runOut(`"name: {/sv_SE.person.last}"`, "--seed", "1")
|
||||
if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
|
||||
t.Fatalf("inline JSON string = %d, %q, stderr %q", code, out, errb)
|
||||
}
|
||||
code, out, errb = runOut("{digits(1)}\n", "--seed", "1", "-n", "2")
|
||||
if code != 0 || len(strings.Split(strings.TrimRight(out, "\n"), "\n")) != 2 {
|
||||
t.Fatalf("inline template with --repeat = %d, %q, stderr %q", code, out, errb)
|
||||
}
|
||||
code, out, errb = runOut("{/sv_SE.person} hihi", "--seed", "1")
|
||||
if code != 0 || !strings.HasSuffix(out, " hihi") || strings.Contains(out, "{") {
|
||||
t.Fatalf("a reference and text = %d, %q, stderr %q", code, out, errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFormatTakesOnlyALoneReferenceAsARecord(t *testing.T) {
|
||||
for _, stdin := range []string{"{//sv_SE.person}", "{/sv_SE.person|/misc.uuid}", "{/sv_SE.person as p}", "{/}", "{/.sv_SE.person}"} {
|
||||
if code, out, errb := runOut(stdin, "--format", "json"); code != 2 || out != "" {
|
||||
t.Errorf("run(%q, --format json) = %d, %q, %q; want misuse", stdin, code, out, errb)
|
||||
}
|
||||
}
|
||||
if code, out, errb := runOut(`{"format":"x"}`); code != 0 || out != "x" {
|
||||
t.Errorf(`run({"format":"x"}) = %d, %q, %q; want x`, code, out, errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunTemplateMisuse(t *testing.T) {
|
||||
for stdin, want := range map[string]string{
|
||||
"{bad": "unterminated",
|
||||
"name: {/no.such.path}": "no entry",
|
||||
"a } b": "}}",
|
||||
"{//sv_SE.person}": "write {/sv_SE.person}",
|
||||
} {
|
||||
code, out, errb := runOut(stdin, "--seed", "1")
|
||||
if code != 2 || out != "" || !strings.Contains(errb, "try 'fejkdata --help'") || !strings.Contains(errb, want) {
|
||||
t.Errorf("run(%q) = %d, %q, %q; want misuse naming %q and --help", stdin, code, out, errb, want)
|
||||
}
|
||||
if strings.Contains(errb, "fejkdata: fejkdata:") {
|
||||
t.Errorf("run(%q) doubled the program prefix: %q", stdin, errb)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNoShippedData(t *testing.T) {
|
||||
code, list, errb := runOut("", "--no-shipped-data", "-d", misc, "--list")
|
||||
if code != 0 {
|
||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
if strings.Contains(list, "sv_SE") || !strings.Contains(list, "uuid\n") {
|
||||
t.Errorf("--no-shipped-data --list = %q, want only the given dir", list)
|
||||
}
|
||||
code, out, _ := runOut("{/uuid}", "--no-shipped-data", "-d", misc, "-s", "3")
|
||||
if code != 0 || strings.TrimSpace(out) == "" {
|
||||
t.Errorf("run = %d, out=%q", code, out)
|
||||
}
|
||||
if code, out, errb := runOut("{digits(3)}", "--no-shipped-data"); code != 0 || len(out) != 3 {
|
||||
t.Errorf("--no-shipped-data alone = %d, %q, %q, want a template reading no data rendered", code, out, errb)
|
||||
}
|
||||
if code, _, errb := runOut("{/sv_SE.person}", "--no-shipped-data"); code != 2 || !strings.HasSuffix(errb, "no data is loaded\ntry 'fejkdata --help'\n") {
|
||||
t.Errorf("--no-shipped-data alone = %d, %q, want a reference to nothing refused", code, errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRepeatOfZeroWritesNoRecord(t *testing.T) {
|
||||
dir := recordDir(t)
|
||||
for _, c := range []struct {
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{nil, ""},
|
||||
{[]string{"--format", "json"}, "[]\n"},
|
||||
{[]string{"--format", "ndjson"}, ""},
|
||||
{[]string{"--format", "csv"}, "first,last\n"},
|
||||
{[]string{"--format", "sql"}, ""},
|
||||
} {
|
||||
if code, out, errb := runOut("{/users}", append(c.args, "-n", "0", "-d", dir)...); code != 0 || out != c.want {
|
||||
t.Errorf("run(-n 0 %v) = %d, %q, %q, want %q", c.args, code, out, errb, c.want)
|
||||
}
|
||||
}
|
||||
if code, _, _ := runOut("{/nope}", "-n", "0", "--format", "json", "-d", dir); code != 1 {
|
||||
t.Errorf("run(-n 0) of a reference to nothing = %d, want 1", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRepeatIsBounded(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{"--repeat", "9223372036854775807"},
|
||||
{"--repeat", "1048577"},
|
||||
{"-n", "99999999999"},
|
||||
} {
|
||||
code, out, errb := runOut("{/sv_SE.word}", args...)
|
||||
if code != 2 || out != "" || !strings.Contains(errb, "0..1048576") {
|
||||
t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb)
|
||||
}
|
||||
}
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, out, errb := runOut("{/x}", "--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "")
|
||||
if code != 0 || len(out) != 1048576 {
|
||||
t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunIgnoresAFlagWithNoEffect(t *testing.T) {
|
||||
dir := recordDir(t)
|
||||
for _, c := range []struct {
|
||||
stdin string
|
||||
base, ignored []string
|
||||
}{
|
||||
{"", []string{"--list"}, []string{"-n", "3"}},
|
||||
{"", []string{"--list"}, []string{"--separator", ","}},
|
||||
{"", []string{"--list"}, []string{"--format", "json"}},
|
||||
{"", []string{"--list"}, []string{"--table", "t"}},
|
||||
{"{/users}", nil, []string{"--table", "t"}},
|
||||
{"{/users}", []string{"--format", "json"}, []string{"--table", "t"}},
|
||||
} {
|
||||
args := append([]string{"--seed", "1", "--data-path", dir}, c.base...)
|
||||
_, want, _ := runOut(c.stdin, args...)
|
||||
code, got, errb := runOut(c.stdin, append(args, c.ignored...)...)
|
||||
if code != 0 || want == "" || got != want {
|
||||
t.Errorf("run(%v + %v) = %d, %q, %q; want what %v alone prints, %q", c.base, c.ignored, code, got, errb, c.base, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReadsABareNumberAsAFormatString(t *testing.T) {
|
||||
if code, out, errb := runOut("42\n", "-n", "3"); code != 0 || out != "42\n42\n42\n" {
|
||||
t.Errorf("echo 42 | fejkdata -n 3 = %d, %q, %q, want 42 on three lines", code, out, errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRendersAnyJSONValue(t *testing.T) {
|
||||
for stdin, want := range map[string]string{"42": "42", "42\n": "42\n", "true\n": "true\n", " 7 ": " 7 ", "null\n": "", " null ": ""} {
|
||||
if code, out, errb := runOut(stdin); code != 0 || out != want {
|
||||
t.Errorf("run(%q) = %d, %q, %q, want %q", stdin, code, out, errb, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUnknownFlagIsNamedByRune(t *testing.T) {
|
||||
code, _, errb := runOut("{/sv_SE.word}", "-ä")
|
||||
if code != 2 || !strings.Contains(errb, "unknown flag -ä") {
|
||||
t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunEmptyDataPathIsNamed(t *testing.T) {
|
||||
code, _, errb := runOut("{/sv_SE.word}", "--data-path=")
|
||||
if code != 1 || !strings.Contains(errb, "empty") {
|
||||
t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb)
|
||||
}
|
||||
}
|
||||
|
||||
func recordDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
content := `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"]}`
|
||||
if err := os.WriteFile(filepath.Join(dir, "users.json"), []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestRunRecordJSON(t *testing.T) {
|
||||
code, out, errb := runOut("{/users}", "--seed", "1", "--format", "json", "--repeat", "2", "--data-path", recordDir(t))
|
||||
if code != 0 {
|
||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
var arr []map[string]string
|
||||
if err := json.Unmarshal([]byte(out), &arr); err != nil {
|
||||
t.Fatalf("json output is not one JSON array: %v\n%q", err, out)
|
||||
}
|
||||
if len(arr) != 2 || len(arr[0]) != 2 || arr[0]["first"] == "" || arr[0]["last"] == "" {
|
||||
t.Fatalf("json output = %q, want an array of two records with first and last columns", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRecordNDJSON(t *testing.T) {
|
||||
code, out, errb := runOut("{/users}", "--seed", "1", "--format", "ndjson", "--repeat", "2", "--data-path", recordDir(t))
|
||||
if code != 0 {
|
||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
|
||||
if len(lines) != 2 {
|
||||
t.Fatalf("ndjson output = %d lines %q, want one object per line", len(lines), out)
|
||||
}
|
||||
for _, line := range lines {
|
||||
var m map[string]string
|
||||
if err := json.Unmarshal([]byte(line), &m); err != nil || len(m) != 2 {
|
||||
t.Fatalf("ndjson line %q is not one object: %v", line, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRecordCSVRoundTrips(t *testing.T) {
|
||||
code, out, errb := runOut("{/users}\r\n", "--seed", "1", "--format", "csv", "--repeat", "3", "--data-path", recordDir(t))
|
||||
if code != 0 {
|
||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
|
||||
if len(lines) != 4 {
|
||||
t.Fatalf("csv output = %d lines %q, want a header and 3 rows", len(lines), out)
|
||||
}
|
||||
if lines[0] != "first,last" {
|
||||
t.Fatalf("csv header = %q, want first,last", lines[0])
|
||||
}
|
||||
r, err := csv.NewReader(strings.NewReader(out)).ReadAll()
|
||||
if err != nil {
|
||||
t.Fatalf("csv output is not valid CSV: %v", err)
|
||||
}
|
||||
if len(r) != 4 || len(r[0]) != 2 {
|
||||
t.Fatalf("csv parsed to %v, want 4 records of 2 fields", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRecordSQL(t *testing.T) {
|
||||
code, out, errb := runOut("{/users}", "--seed", "1", "--format", "sql", "--repeat", "2", "--table", "people", "--data-path", recordDir(t))
|
||||
if code != 0 {
|
||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
|
||||
if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) {
|
||||
t.Fatalf("sql output = %q, want two INSERT INTO people statements", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRecordDefaultTable(t *testing.T) {
|
||||
code, out, errb := runOut("{/users}", "--seed", "1", "--format", "sql", "--data-path", recordDir(t))
|
||||
if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
|
||||
t.Fatalf("default table = %d, %q, want the reference's last segment; stderr %q", code, out, errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRecordOfAQuotedLoneReference(t *testing.T) {
|
||||
code, out, errb := runOut(`"{/users}"`, "--seed", "1", "--format", "sql", "--data-path", recordDir(t))
|
||||
if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
|
||||
t.Fatalf("a JSON string holding {/users} = %d, %q, %q; want the record users", code, out, errb)
|
||||
}
|
||||
code, out, errb = runOut("{/users} ", "--format", "csv", "--data-path", recordDir(t))
|
||||
if code != 2 || out != "" || !strings.Contains(errb, "whitespace around {/users}") {
|
||||
t.Errorf("{/users} and a space = %d, %q, %q; want misuse naming the whitespace", code, out, errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRecordInlineTemplate(t *testing.T) {
|
||||
code, out, errb := runOut(`{"format":"{x}","x":["a","b"]}`, "--seed", "1", "--format", "json")
|
||||
if code != 0 {
|
||||
t.Fatalf("inline record = %d, stderr=%q", code, errb)
|
||||
}
|
||||
var arr []map[string]string
|
||||
if err := json.Unmarshal([]byte(out), &arr); err != nil || len(arr) != 1 || (arr[0]["x"] != "a" && arr[0]["x"] != "b") {
|
||||
t.Fatalf("inline record json = %q, want one record with a column x (err %v)", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRecordMisuse(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{[]string{"--format", "yaml"}, "--format takes csv, json, ndjson, sql or text"},
|
||||
{[]string{"--format", "sql", "--table", ""}, "--table names the INSERT target"},
|
||||
{[]string{"--format", "csv", "--separator", ";"}, `--separator joins text renders; --format csv always separates fields with ","`},
|
||||
{[]string{"--format", "json", "--separator", ","}, "--separator joins text renders"},
|
||||
{[]string{"--format", "ndjson", "--separator", ","}, "--separator joins text renders"},
|
||||
{[]string{"--format", "sql", "--separator", ","}, "--separator joins text renders"},
|
||||
} {
|
||||
code, out, errb := runOut("{/users}", c.args...)
|
||||
if code != 2 || out != "" || !strings.Contains(errb, c.want) {
|
||||
t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRecordOnABareValueIsRuntimeError(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "word.json"), []byte(`["a","b"]`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, _, errb := runOut("{/word}", "--format", "json", "--data-path", dir)
|
||||
if code != 1 {
|
||||
t.Fatalf("record on a choice = %d, want exit 1 (runtime error), stderr %q", code, errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRecordRejectsATopLevelRepeat(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
content := `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`
|
||||
if err := os.WriteFile(filepath.Join(dir, "rep.json"), []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, _, errb := runOut("{/rep}", "--format", "json", "--data-path", dir)
|
||||
if code != 1 || !strings.Contains(errb, "carries repeat 3") {
|
||||
t.Errorf("record on a repeating path = %d, %q; want exit 1 naming the repeat", code, errb)
|
||||
}
|
||||
code, _, errb = runOut(content, "--format", "json")
|
||||
if code != 2 || !strings.Contains(errb, "carries repeat 3") {
|
||||
t.Errorf("record on a repeating inline template = %d, %q; want exit 2 naming the repeat", code, errb)
|
||||
}
|
||||
if code, out, _ := runOut("{/rep}", "--seed", "1", "--data-path", dir); code != 0 || strings.TrimSpace(out) != "x-|x-|x-" {
|
||||
t.Errorf("text view = %d, %q; want the repeat still composed", code, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNumbersFollowTheShell(t *testing.T) {
|
||||
_, want, _ := runOut("{/sv_SE.word}", "--seed", "7", "--repeat", "3")
|
||||
code, got, errb := runOut("{/sv_SE.word}", "--seed", "007", "--repeat", "+3")
|
||||
if code != 0 || got != want {
|
||||
t.Errorf("run(--seed 007 --repeat +3) = %d, %q, stderr %q; want the same as --seed 7 --repeat 3 %q", code, got, errb, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunRecordOfAFieldlessPath pins the way out of asking a category of one value
|
||||
// for columns: the record to write, and the quoting a shell needs to take it. The
|
||||
// template is well-formed and only the data cannot serve it, so the code is 1.
|
||||
func TestRunRecordOfAFieldlessPath(t *testing.T) {
|
||||
code, out, errb := runOut("{/sv_SE.personnummer}", "--format", "csv")
|
||||
if code != 1 {
|
||||
t.Fatalf("run(--format csv) of {/sv_SE.personnummer} = %d, want 1", code)
|
||||
}
|
||||
for _, want := range []string{`{"format":"","personnummer":"{/sv_SE.personnummer}"}`, "single quotes"} {
|
||||
if !strings.Contains(errb, want) {
|
||||
t.Errorf("stderr = %q, want it to name %s", errb, want)
|
||||
}
|
||||
}
|
||||
if out != "" {
|
||||
t.Errorf("stdout = %q, want nothing", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADataPathCategoryFailingAtFirstReachExits1(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for name, data := range map[string]string{
|
||||
".fejkdata.json": `{"index": {"bad": {"paths": [""]}}}`,
|
||||
"bad.json": `"{/nope}"`,
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, args := range [][]string{{"-d", dir}, {"-d", dir, "--format", "json"}} {
|
||||
if code, _, errb := runOut("{/bad}", args...); code != 1 || !strings.Contains(errb, "{/nope}") {
|
||||
t.Errorf("%v on a category failing to load = %d, %q; want exit 1 naming the reference", args, code, errb)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/larvit/fejkdata/internal/grammar"
|
||||
)
|
||||
|
||||
// grammarLoneReference is the path a format reads when the grammar parses it as one path
|
||||
// read of one reference from the root, and nothing else.
|
||||
func grammarLoneReference(text string) (string, bool) {
|
||||
toks, err := grammar.ParseFormat(text)
|
||||
if err != nil || len(toks) != 1 || toks[0].Kind != grammar.PathRead || len(toks[0].Arms) != 1 {
|
||||
return "", false
|
||||
}
|
||||
sigil, path, err := grammar.RefShape(toks[0].Arms[0])
|
||||
if err != nil || sigil != "/" {
|
||||
return "", false
|
||||
}
|
||||
return path, true
|
||||
}
|
||||
|
||||
func TestLoneReferenceAgreesWithTheGrammar(t *testing.T) {
|
||||
for _, text := range []string{
|
||||
" {/x}",
|
||||
"x",
|
||||
"{.x}",
|
||||
"{/.x}",
|
||||
"{//x}",
|
||||
"{/a...b}",
|
||||
"{/a..b}",
|
||||
"{/a..}",
|
||||
"{/a.[x]}",
|
||||
"{/a.}",
|
||||
"{/a|/b}",
|
||||
"{/a}{/b}",
|
||||
"{/misc.currency[US Dollar (Next day)]}",
|
||||
"{/sv_SE.person}",
|
||||
"{/t[a as b].c}",
|
||||
"{/t[f(x)]}",
|
||||
"{/x as p}",
|
||||
"{/x(1)}",
|
||||
"{/x}}",
|
||||
"{/}",
|
||||
} {
|
||||
path, lone := loneReference(text)
|
||||
want, wantLone := grammarLoneReference(text)
|
||||
if lone != wantLone || path != want {
|
||||
t.Errorf("loneReference(%q) = %q, %v; the grammar reads %q, %v", text, path, lone, want, wantLone)
|
||||
}
|
||||
}
|
||||
}
|
||||
+270
-5
@@ -1,12 +1,277 @@
|
||||
// Command fejkdata renders the template on its stdin with fake values from every shipped
|
||||
// module; package cli documents it.
|
||||
// Command fejkdata prints fake values from the shipped data and any directories
|
||||
// layered over it.
|
||||
//
|
||||
// fejkdata sv_SE.person # a full person
|
||||
// fejkdata sv_SE.person.last # just the surname
|
||||
// fejkdata --data-path ./mydata sv_SE.person # layer custom data; the last dir wins
|
||||
// fejkdata --seed 42 sv_SE.address
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/larvit/fejkdata/cli"
|
||||
"github.com/larvit/fejkdata/data"
|
||||
"gitea.larvit.se/larvit/fejkdata"
|
||||
)
|
||||
|
||||
func main() { os.Exit(cli.Main(data.Modules()...)) }
|
||||
const usage = `Usage: fejkdata [flags] <path>
|
||||
|
||||
<path> a category, or a dotted path into one (person, person.last)
|
||||
|
||||
-d, --data-path D a data directory to layer over the shipped data (repeatable; last wins on a clash)
|
||||
-h, --help print this help, then exit
|
||||
--list list the paths the data offers, then exit
|
||||
--no-shipped-data load only the --data-path directories
|
||||
-n, --repeat N render the path N times (default 1)
|
||||
-s, --seed N seed for reproducible output
|
||||
--separator S string between repeated values (default newline)
|
||||
--version print the version, then exit
|
||||
|
||||
Flags may come before or after <path>; -- ends the flags. A short flag's value
|
||||
attaches or follows (-n3, -n 3); short flags bundle (-hn 3).
|
||||
`
|
||||
|
||||
type invocation struct {
|
||||
dirs []string
|
||||
help bool
|
||||
list bool
|
||||
noShipped bool
|
||||
paths []string
|
||||
repeat int
|
||||
seed uint64
|
||||
seeded bool
|
||||
separator string
|
||||
version bool
|
||||
}
|
||||
|
||||
type flagDef struct {
|
||||
long string
|
||||
short string
|
||||
value bool
|
||||
set func(*invocation, string) error
|
||||
}
|
||||
|
||||
var flagDefs = []flagDef{
|
||||
{"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }},
|
||||
{"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }},
|
||||
{"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }},
|
||||
{"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }},
|
||||
{"repeat", "n", true, func(in *invocation, v string) error {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 1 {
|
||||
return fmt.Errorf("--repeat needs a positive integer, got %q", v)
|
||||
}
|
||||
in.repeat = n
|
||||
return nil
|
||||
}},
|
||||
{"seed", "s", true, func(in *invocation, v string) error {
|
||||
n, err := strconv.ParseUint(v, 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("--seed needs an unsigned integer, got %q", v)
|
||||
}
|
||||
in.seed, in.seeded = n, true
|
||||
return nil
|
||||
}},
|
||||
{"separator", "", true, func(in *invocation, v string) error { in.separator = v; return nil }},
|
||||
{"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }},
|
||||
}
|
||||
|
||||
func flagByLong(name string) *flagDef {
|
||||
for i := range flagDefs {
|
||||
if flagDefs[i].long == name {
|
||||
return &flagDefs[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func flagByShort(name string) *flagDef {
|
||||
for i := range flagDefs {
|
||||
if flagDefs[i].short == name {
|
||||
return &flagDefs[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// flagArg is one flag as written: its definition and the value attached to it,
|
||||
// if any.
|
||||
type flagArg struct {
|
||||
def *flagDef
|
||||
value string
|
||||
inline bool
|
||||
}
|
||||
|
||||
// splitFlags reads one argv element as flags: --name or --name=value, or -abc
|
||||
// where each letter is a short flag and the first that takes a value takes the
|
||||
// rest of the element.
|
||||
func splitFlags(arg string) ([]flagArg, error) {
|
||||
if strings.HasPrefix(arg, "--") {
|
||||
name, value, inline := strings.Cut(arg[2:], "=")
|
||||
def := flagByLong(name)
|
||||
if def == nil {
|
||||
return nil, fmt.Errorf("unknown flag --%s", name)
|
||||
}
|
||||
return []flagArg{{def, value, inline}}, nil
|
||||
}
|
||||
if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil {
|
||||
return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name)
|
||||
}
|
||||
var flags []flagArg
|
||||
letters := arg[1:]
|
||||
for i := 0; i < len(letters); i++ {
|
||||
letter := letters[i : i+1]
|
||||
def := flagByShort(letter)
|
||||
if def == nil {
|
||||
return nil, fmt.Errorf("unknown flag -%s", letter)
|
||||
}
|
||||
if !def.value {
|
||||
flags = append(flags, flagArg{def: def})
|
||||
continue
|
||||
}
|
||||
rest := letters[i+1:]
|
||||
if strings.HasPrefix(rest, "=") {
|
||||
return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s",
|
||||
letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:])
|
||||
}
|
||||
return append(flags, flagArg{def, rest, rest != ""}), nil
|
||||
}
|
||||
return flags, nil
|
||||
}
|
||||
|
||||
func parseArgs(argv []string) (invocation, error) {
|
||||
in := invocation{repeat: 1, separator: "\n"}
|
||||
for i := 0; i < len(argv); i++ {
|
||||
arg := argv[i]
|
||||
if arg == "--" {
|
||||
in.paths = append(in.paths, argv[i+1:]...)
|
||||
return in, nil
|
||||
}
|
||||
if len(arg) < 2 || arg[0] != '-' {
|
||||
in.paths = append(in.paths, arg)
|
||||
continue
|
||||
}
|
||||
flags, err := splitFlags(arg)
|
||||
if err != nil {
|
||||
return in, err
|
||||
}
|
||||
for _, fl := range flags {
|
||||
if !fl.def.value {
|
||||
if fl.inline {
|
||||
return in, fmt.Errorf("--%s takes no value", fl.def.long)
|
||||
}
|
||||
_ = fl.def.set(&in, "")
|
||||
continue
|
||||
}
|
||||
value := fl.value
|
||||
if !fl.inline {
|
||||
if i++; i >= len(argv) {
|
||||
return in, fmt.Errorf("--%s needs a value", fl.def.long)
|
||||
}
|
||||
value = argv[i]
|
||||
}
|
||||
if err := fl.def.set(&in, value); err != nil {
|
||||
return in, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return in, nil
|
||||
}
|
||||
|
||||
// check rejects a flag combination that cannot run.
|
||||
func (in invocation) check() error {
|
||||
if in.list && len(in.paths) > 0 {
|
||||
return errors.New("--list takes no path")
|
||||
}
|
||||
if !in.list && len(in.paths) != 1 {
|
||||
return fmt.Errorf("expected one path, got %d", len(in.paths))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (in invocation) options() []fejkdata.Option {
|
||||
var opts []fejkdata.Option
|
||||
if in.noShipped {
|
||||
opts = append(opts, fejkdata.WithoutShippedData())
|
||||
}
|
||||
for _, dir := range in.dirs {
|
||||
opts = append(opts, fejkdata.WithDataPath(dir))
|
||||
}
|
||||
if in.seeded {
|
||||
opts = append(opts, fejkdata.WithSeed(in.seed))
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
// values renders the path repeat times, joined by the separator.
|
||||
func (in invocation) values(f *fejkdata.Generator) (string, error) {
|
||||
vals := make([]string, in.repeat)
|
||||
for i := range vals {
|
||||
v, err := f.Fake(in.paths[0])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
vals[i] = v
|
||||
}
|
||||
return strings.Join(vals, in.separator), nil
|
||||
}
|
||||
|
||||
func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) }
|
||||
|
||||
// run returns the exit code: 0 ok, 1 runtime error, 2 misuse.
|
||||
func run(args []string, stdout, stderr io.Writer) int {
|
||||
in, err := parseArgs(args)
|
||||
if err != nil {
|
||||
return misuse(stderr, err)
|
||||
}
|
||||
if in.help {
|
||||
fmt.Fprint(stdout, usage)
|
||||
return 0
|
||||
}
|
||||
if in.version {
|
||||
fmt.Fprintln(stdout, "fejkdata "+buildVersion())
|
||||
return 0
|
||||
}
|
||||
if err := in.check(); err != nil {
|
||||
return misuse(stderr, err)
|
||||
}
|
||||
f, err := fejkdata.New(in.options()...)
|
||||
if errors.Is(err, fejkdata.ErrNoData) {
|
||||
return misuse(stderr, errors.New("--no-shipped-data needs at least one --data-path"))
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
if in.list {
|
||||
for _, p := range f.List() {
|
||||
fmt.Fprintln(stdout, p)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
out, err := in.values(f)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
fmt.Fprintln(stdout, out)
|
||||
return 0
|
||||
}
|
||||
|
||||
func misuse(stderr io.Writer, err error) int {
|
||||
fmt.Fprintf(stderr, "fejkdata: %v\ntry 'fejkdata --help'\n", err)
|
||||
return 2
|
||||
}
|
||||
|
||||
// buildVersion is the module version go install stamps into the binary, or "devel".
|
||||
func buildVersion() string {
|
||||
if info, ok := debug.ReadBuildInfo(); ok && info.Main.Version != "" {
|
||||
return info.Main.Version
|
||||
}
|
||||
return "devel"
|
||||
}
|
||||
|
||||
+340
-30
@@ -3,45 +3,355 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
if os.Getenv("FEJKDATA_RUN_MAIN") == "1" {
|
||||
main()
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
const (
|
||||
svSE = "../../data/sv_SE"
|
||||
enUS = "../../data/en_US"
|
||||
)
|
||||
|
||||
func runOut(args ...string) (int, string, string) {
|
||||
var out, errb bytes.Buffer
|
||||
code := run(args, &out, &errb)
|
||||
return code, out.String(), errb.String()
|
||||
}
|
||||
|
||||
func TestMainRendersTheShippedData(t *testing.T) {
|
||||
cmd := exec.Command(os.Args[0], "--seed", "1")
|
||||
cmd.Env = append(os.Environ(), "FEJKDATA_RUN_MAIN=1")
|
||||
cmd.Stdin = strings.NewReader("{/misc.territory[SE].alpha2} {/en_US.person.last} {/sv_SE.address.locality}")
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
out, err := cmd.Output()
|
||||
if fields := strings.Fields(string(out)); err != nil || len(fields) < 3 || fields[0] != "SE" {
|
||||
t.Errorf("fejkdata = %v, %q, %q, want SE followed by a last name and a locality", err, out, stderr.String())
|
||||
func TestRunOutputsValue(t *testing.T) {
|
||||
code, out, errb := runOut("--data-path", svSE, "person")
|
||||
if code != 0 {
|
||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
if strings.TrimSpace(out) == "" {
|
||||
t.Fatalf("empty output, stderr=%q", errb)
|
||||
}
|
||||
if !strings.HasSuffix(out, "\n") {
|
||||
t.Errorf("output should end with newline, got %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMainWithoutASeedDrawsANewSeedEachRun(t *testing.T) {
|
||||
var outs [2]string
|
||||
for i := range outs {
|
||||
cmd := exec.Command(os.Args[0])
|
||||
cmd.Env = append(os.Environ(), "FEJKDATA_RUN_MAIN=1")
|
||||
cmd.Stdin = strings.NewReader("{uuid()}")
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
out, err := cmd.Output()
|
||||
if err != nil || len(out) == 0 {
|
||||
t.Fatalf("fejkdata without --seed = %v, %q, %q", err, out, stderr.String())
|
||||
func TestRunDotPath(t *testing.T) {
|
||||
code, full, _ := runOut("--seed", "7", "--data-path", svSE, "person")
|
||||
if code != 0 {
|
||||
t.Fatalf("person run = %d", code)
|
||||
}
|
||||
code, last, errb := runOut("--seed", "7", "--data-path", svSE, "person.last")
|
||||
if code != 0 {
|
||||
t.Fatalf("person.last run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
if strings.TrimSpace(last) == "" {
|
||||
t.Fatal("empty person.last output")
|
||||
}
|
||||
if last == full {
|
||||
t.Errorf("person.last %q should differ from person %q", last, full)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSeedSpellings(t *testing.T) {
|
||||
_, want, _ := runOut("--seed", "42", "--data-path", svSE, "address")
|
||||
for _, args := range [][]string{
|
||||
{"--seed=42", "--data-path", svSE, "address"},
|
||||
{"-s", "42", "-d", svSE, "address"},
|
||||
{"--data-path", svSE, "address", "--seed", "42"},
|
||||
} {
|
||||
code, got, errb := runOut(args...)
|
||||
if code != 0 {
|
||||
t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("run(%v) = %q, want %q", args, got, want)
|
||||
}
|
||||
outs[i] = string(out)
|
||||
}
|
||||
if outs[0] == outs[1] {
|
||||
t.Errorf("two runs without --seed both printed %q, want a new seed each run", outs[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunShortFlagValues(t *testing.T) {
|
||||
_, want, _ := runOut("--seed", "42", "--data-path", svSE, "address")
|
||||
for _, args := range [][]string{
|
||||
{"-s42", "-d", svSE, "address"},
|
||||
{"-s", "42", "-d" + svSE, "address"},
|
||||
{"-d", svSE, "address", "-s42"},
|
||||
} {
|
||||
code, got, errb := runOut(args...)
|
||||
if code != 0 {
|
||||
t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("run(%v) = %q, want %q", args, got, want)
|
||||
}
|
||||
}
|
||||
_, three, _ := runOut("-s", "1", "-n3", "-d", svSE, "word")
|
||||
if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 {
|
||||
t.Errorf("-n3 gave %d lines: %q", len(lines), three)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{[]string{"-s=42", "-d", svSE, "address"}, "-s42"},
|
||||
{[]string{"-s=42", "-d", svSE, "address"}, "--seed=42"},
|
||||
{[]string{"-d=" + svSE, "address"}, "--data-path="},
|
||||
{[]string{"-nd", "3", "-d", svSE, "word"}, `--repeat needs a positive integer, got "d"`},
|
||||
{[]string{"--seed=", "-d", svSE, "word"}, `--seed needs an unsigned integer, got ""`},
|
||||
} {
|
||||
code, out, errb := runOut(c.args...)
|
||||
if code != 2 || out != "" {
|
||||
t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out)
|
||||
}
|
||||
if !strings.Contains(errb, c.want) {
|
||||
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
|
||||
}
|
||||
}
|
||||
code, out, _ := runOut("-hd", svSE)
|
||||
if code != 0 || !strings.Contains(out, "Usage") {
|
||||
t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDoubleDashEndsFlags(t *testing.T) {
|
||||
code, out, errb := runOut("--data-path", svSE, "--", "person")
|
||||
if code != 0 || strings.TrimSpace(out) == "" {
|
||||
t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
|
||||
}
|
||||
code, _, errb = runOut("--data-path", svSE, "--", "--list")
|
||||
if code != 1 || !strings.Contains(errb, "--list") {
|
||||
t.Errorf("after --, --list should be a path: code %d, stderr %q", code, errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSingleDashLongIsRejected(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{[]string{"-seed", "42", "-d", svSE, "person"}, "use --seed"},
|
||||
{[]string{"-seed=42", "-d", svSE, "person"}, "use --seed"},
|
||||
{[]string{"-data-path", svSE, "person"}, "use --data-path"},
|
||||
{[]string{"-list", "-d", svSE}, "use --list"},
|
||||
{[]string{"--nope", "-d", svSE, "person"}, "unknown flag --nope"},
|
||||
{[]string{"-x", "-d", svSE, "person"}, "unknown flag -x"},
|
||||
} {
|
||||
code, _, errb := runOut(c.args...)
|
||||
if code != 2 {
|
||||
t.Errorf("run(%v) = %d, want 2", c.args, code)
|
||||
}
|
||||
if !strings.Contains(errb, c.want) {
|
||||
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFlagValues(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{[]string{"--seed", "-d", svSE, "person"}, `--seed needs an unsigned integer, got "-d"`},
|
||||
{[]string{"-d", svSE, "person", "--seed"}, "--seed needs a value"},
|
||||
{[]string{"--seed", "x", "-d", svSE, "person"}, "--seed"},
|
||||
{[]string{"--list=1", "-d", svSE}, "--list takes no value"},
|
||||
{[]string{"--repeat", "0", "-d", svSE, "person"}, "--repeat"},
|
||||
{[]string{"-n", "-1", "-d", svSE, "person"}, "--repeat"},
|
||||
} {
|
||||
code, _, errb := runOut(c.args...)
|
||||
if code != 2 {
|
||||
t.Errorf("run(%v) = %d, want 2", c.args, code)
|
||||
}
|
||||
if !strings.Contains(errb, c.want) {
|
||||
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRepeat(t *testing.T) {
|
||||
code, out, errb := runOut("--seed", "1", "--repeat", "3", "--data-path", svSE, "word")
|
||||
if code != 0 {
|
||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
|
||||
if len(lines) != 3 {
|
||||
t.Errorf("repeat=3 gave %d lines: %q", len(lines), out)
|
||||
}
|
||||
_, short, _ := runOut("--seed", "1", "-n", "3", "-d", svSE, "word")
|
||||
if short != out {
|
||||
t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSeparator(t *testing.T) {
|
||||
code, out, errb := runOut("--repeat", "3", "--separator", ",", "--data-path", svSE, "word")
|
||||
if code != 0 {
|
||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
if n := strings.Count(out, "\n"); n != 1 {
|
||||
t.Errorf("want one trailing newline, got %d: %q", n, out)
|
||||
}
|
||||
if !strings.Contains(out, ",") {
|
||||
t.Errorf("values should be comma-joined: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRepeatAdvancesRNG(t *testing.T) {
|
||||
code, out, errb := runOut("--seed", "1", "--repeat", "5", "--data-path", svSE, "person")
|
||||
if code != 0 {
|
||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
uniq := map[string]bool{}
|
||||
for _, l := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
|
||||
uniq[l] = true
|
||||
}
|
||||
if len(uniq) < 2 {
|
||||
t.Errorf("repeat should vary output, all identical: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMisuse(t *testing.T) {
|
||||
for _, args := range [][]string{{}, {"--data-path", svSE}, {"-d", svSE, "person", "word"}, {"-d", svSE, "--list", "person"}, {"--no-shipped-data", "sv_SE.person"}} {
|
||||
code, out, errb := runOut(args...)
|
||||
if code != 2 {
|
||||
t.Errorf("run(%v) = %d, want 2", args, code)
|
||||
}
|
||||
if !strings.Contains(errb, "try 'fejkdata --help'") {
|
||||
t.Errorf("run(%v) stderr = %q, want a pointer to --help", args, errb)
|
||||
}
|
||||
if out != "" {
|
||||
t.Errorf("run(%v) stdout = %q, want nothing", args, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMultipleDataPaths(t *testing.T) {
|
||||
code, out, errb := runOut("-d", enUS, "--data-path", svSE, "person")
|
||||
if code != 0 {
|
||||
t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb)
|
||||
}
|
||||
if strings.TrimSpace(out) == "" {
|
||||
t.Fatal("empty output for multi-dir run")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUnknownCategoryFails(t *testing.T) {
|
||||
code, _, errb := runOut("--data-path", svSE, "nope")
|
||||
if code != 1 {
|
||||
t.Fatalf("run = %d, want 1", code)
|
||||
}
|
||||
if !strings.Contains(errb, "nope") {
|
||||
t.Errorf("stderr %q should name the unknown category", errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunList(t *testing.T) {
|
||||
code, out, errb := runOut("--data-path", svSE, "--list")
|
||||
if code != 0 {
|
||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
for _, want := range []string{"person", "person.last", "address"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("list output missing %q:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunHelpOnStdout(t *testing.T) {
|
||||
for _, h := range []string{"-h", "--help"} {
|
||||
code, out, errb := runOut(h)
|
||||
if code != 0 {
|
||||
t.Errorf("%s = %d, want 0", h, code)
|
||||
}
|
||||
if !strings.Contains(out, "Usage") || errb != "" {
|
||||
t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunVersion(t *testing.T) {
|
||||
code, out, errb := runOut("--version")
|
||||
if code != 0 {
|
||||
t.Fatalf("--version = %d, stderr=%q", code, errb)
|
||||
}
|
||||
version, ok := strings.CutPrefix(out, "fejkdata ")
|
||||
if !ok || strings.TrimSpace(version) == "" {
|
||||
t.Errorf("--version = %q, want the command name and a version on stdout", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMissingDirFails(t *testing.T) {
|
||||
code, _, errb := runOut("--data-path", "../../data/nope", "person")
|
||||
if code != 1 {
|
||||
t.Fatalf("run = %d, want 1", code)
|
||||
}
|
||||
if errb == "" {
|
||||
t.Error("want an error message on stderr")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunShippedDataByDefault(t *testing.T) {
|
||||
code, out, errb := runOut("--seed", "1", "sv_SE.person")
|
||||
if code != 0 || strings.TrimSpace(out) == "" {
|
||||
t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
|
||||
}
|
||||
code, list, _ := runOut("--list")
|
||||
if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") {
|
||||
t.Errorf("--list without --data-path = %d, %q", code, list)
|
||||
}
|
||||
code, _, errb = runOut("person")
|
||||
if code != 1 || !strings.Contains(errb, "person") {
|
||||
t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNoShippedData(t *testing.T) {
|
||||
code, list, errb := runOut("--no-shipped-data", "-d", svSE, "--list")
|
||||
if code != 0 {
|
||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
if strings.Contains(list, "en_US") || !strings.Contains(list, "person\n") {
|
||||
t.Errorf("--no-shipped-data --list = %q, want only the given dir", list)
|
||||
}
|
||||
code, out, _ := runOut("--no-shipped-data", "-d", svSE, "-s", "3", "person")
|
||||
if code != 0 || strings.TrimSpace(out) == "" {
|
||||
t.Errorf("run = %d, out=%q", code, out)
|
||||
}
|
||||
code, _, errb = runOut("--no-shipped-data", "sv_SE.person")
|
||||
if code != 2 || !strings.Contains(errb, "--no-shipped-data needs at least one --data-path") {
|
||||
t.Errorf("--no-shipped-data alone = %d, %q, want misuse naming --data-path", code, errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRepeatIsBounded(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{"--repeat", "9223372036854775807", "sv_SE.word"},
|
||||
{"--repeat", "1048577", "sv_SE.word"},
|
||||
{"-n", "99999999999", "sv_SE.word"},
|
||||
} {
|
||||
code, out, errb := runOut(args...)
|
||||
if code != 2 || out != "" || !strings.Contains(errb, "1..1048576") {
|
||||
t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb)
|
||||
}
|
||||
}
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, out, errb := runOut("--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "", "x")
|
||||
if code != 0 || len(out) != 1048576+1 {
|
||||
t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunListTakesNoRepeatOrSeparator(t *testing.T) {
|
||||
for _, args := range [][]string{{"--list", "-n", "3"}, {"--list", "--separator", ","}} {
|
||||
code, _, errb := runOut(args...)
|
||||
if code != 2 || !strings.Contains(errb, "--list takes no") {
|
||||
t.Errorf("run(%v) = %d, %q, want misuse", args, code, errb)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUnknownFlagIsNamedByRune(t *testing.T) {
|
||||
code, _, errb := runOut("-ä", "sv_SE.word")
|
||||
if code != 2 || !strings.Contains(errb, "unknown flag -ä") {
|
||||
t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb)
|
||||
}
|
||||
}
|
||||
|
||||
+19
-18
@@ -1,5 +1,21 @@
|
||||
# Docker-based dev workflow — no local tooling beyond Docker is required.
|
||||
#
|
||||
# docker compose run --rm test
|
||||
# docker compose run --rm vet
|
||||
# docker compose run --rm build
|
||||
#
|
||||
# The full gate — vet, formatting and tests — is `docker build .`, which is what
|
||||
# CI runs.
|
||||
#
|
||||
# Build caches persist in the `gocache` volume, so re-runs are fast and the
|
||||
# host tree stays clean. Commands that rewrite source (fmt, tidy) keep your
|
||||
# ownership when run with `--user "$(id -u):$(id -g)"`.
|
||||
#
|
||||
# GO_VERSION picks the toolchain image (default: latest stable). Test the lowest
|
||||
# supported Go with: GO_VERSION=1.22 docker compose run --rm test
|
||||
|
||||
x-go: &go
|
||||
image: golang:${GO_VERSION:-1.27.2}
|
||||
image: golang:${GO_VERSION:-1.26.4}
|
||||
working_dir: /app
|
||||
volumes:
|
||||
- .:/app
|
||||
@@ -8,7 +24,6 @@ x-go: &go
|
||||
HOME: /cache
|
||||
GOCACHE: /cache/build
|
||||
GOMODCACHE: /cache/mod
|
||||
REPIN: ${REPIN:-}
|
||||
|
||||
services:
|
||||
test:
|
||||
@@ -17,16 +32,12 @@ services:
|
||||
|
||||
cover:
|
||||
<<: *go
|
||||
command: sh -c "go test -coverpkg=./... -coverprofile=/tmp/cover.out ./... && go tool cover -func=/tmp/cover.out | tail -1"
|
||||
command: go test -cover ./...
|
||||
|
||||
bench:
|
||||
<<: *go
|
||||
command: go test -run XXX -bench . -benchmem ./...
|
||||
|
||||
generate:
|
||||
<<: *go
|
||||
command: go generate ./...
|
||||
|
||||
vet:
|
||||
<<: *go
|
||||
command: go vet ./...
|
||||
@@ -35,10 +46,6 @@ services:
|
||||
<<: *go
|
||||
command: go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test .
|
||||
|
||||
vulncheck:
|
||||
<<: *go
|
||||
command: go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan module
|
||||
|
||||
build:
|
||||
<<: *go
|
||||
command: go build ./...
|
||||
@@ -51,16 +58,10 @@ services:
|
||||
<<: *go
|
||||
command: go mod tidy
|
||||
|
||||
# Interactive shell for ad-hoc work: docker compose run --rm dev
|
||||
dev:
|
||||
<<: *go
|
||||
command: sh
|
||||
|
||||
data-import:
|
||||
image: python:3.14.8-slim
|
||||
working_dir: /app
|
||||
volumes:
|
||||
- .:/app
|
||||
entrypoint: python3
|
||||
|
||||
volumes:
|
||||
gocache:
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild data/misc/currency.tsv from datasets/currency-codes (PDDL) and CLDR's symbols (Unicode).
|
||||
|
||||
data-import/currency.py [--source URL_OR_FILE] [--symbols URL_OR_FILE ...] [--cache DIR] [--out FILE]
|
||||
|
||||
The symbols come from the first locale file that has one, narrow symbols before wide.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
|
||||
import source
|
||||
import tsv
|
||||
|
||||
SOURCE = "https://raw.githubusercontent.com/datasets/currency-codes/main/data/codes-all.csv"
|
||||
SYMBOLS = [
|
||||
"https://raw.githubusercontent.com/unicode-org/cldr/main/common/main/en.xml",
|
||||
"https://raw.githubusercontent.com/unicode-org/cldr/main/common/main/root.xml",
|
||||
]
|
||||
OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "currency.tsv"
|
||||
CACHE = Path(__file__).resolve().parent / "cache"
|
||||
COLUMNS = ["code", "decimals", "name", "numeric", "symbol"]
|
||||
|
||||
|
||||
def symbols(xml_texts):
|
||||
"""CLDR's symbol per code: the first locale's narrow symbol, else the first locale's wide one."""
|
||||
narrow, wide = {}, {}
|
||||
for text in xml_texts:
|
||||
for c in ET.fromstring(text).iter("currency"):
|
||||
for s in c.findall("symbol"):
|
||||
into = narrow if s.get("alt") == "narrow" else wide if s.get("alt") is None else None
|
||||
if into is not None:
|
||||
into.setdefault(c.get("type"), s.text)
|
||||
return {**wide, **narrow}
|
||||
|
||||
|
||||
def rows(text, symbol):
|
||||
seen = set()
|
||||
for r in csv.DictReader(io.StringIO(text)):
|
||||
code = r["AlphabeticCode"]
|
||||
if not code or code in seen or r["WithdrawalDate"] or r["Entity"].startswith("ZZ") or not r["MinorUnit"].isdigit():
|
||||
continue
|
||||
seen.add(code)
|
||||
yield {
|
||||
"code": code,
|
||||
"decimals": r["MinorUnit"],
|
||||
"name": r["Currency"],
|
||||
"numeric": r["NumericCode"].zfill(3),
|
||||
"symbol": symbol.get(code, code),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("--cache", default=str(CACHE))
|
||||
p.add_argument("--source", default=SOURCE)
|
||||
p.add_argument("--symbols", nargs="+", default=SYMBOLS)
|
||||
p.add_argument("--out", default=str(OUT))
|
||||
a = p.parse_args()
|
||||
symbol = symbols(source.fetch(s, a.cache, Path(s).name).decode("utf-8") for s in a.symbols)
|
||||
table = rows(source.fetch(a.source, a.cache, "codes-all.csv").decode("utf-8"), symbol)
|
||||
tsv.write(a.out, COLUMNS, sorted(table, key=lambda r: r["code"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,212 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild data/geo/SE/*.tsv from SCB (CC0), GeoNames (CC BY 4.0) and Trafikverket NVDB (CC0).
|
||||
|
||||
TRAFIKVERKET_API_KEY=… data-import/geo-se.py [--key-file FILE] [--cache DIR] [--min-streets N] [--street-share F] [--out DIR]
|
||||
"""
|
||||
import collections
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import geo
|
||||
import source
|
||||
import tsv
|
||||
import xlsx
|
||||
|
||||
CODES = "https://www.scb.se/contentassets/7a89e48960f741e08918e489ea36354a/kommunlankod-2026.xlsx"
|
||||
POPULATION = "https://api.scb.se/OV0104/v1/doris/sv/ssd/START/BE/BE0101/BE0101A/BefolkningNy"
|
||||
POPULATION_QUERY = {
|
||||
"query": [
|
||||
{"code": "Region", "selection": {"filter": "all", "values": ["*"]}},
|
||||
{"code": "ContentsCode", "selection": {"filter": "item", "values": ["BE0101N1"]}},
|
||||
{"code": "Tid", "selection": {"filter": "top", "values": ["1"]}},
|
||||
],
|
||||
"response": {"format": "json"},
|
||||
}
|
||||
TATORTER = "https://geodata.scb.se/geoserver/stat/wfs?service=WFS&version=2.0.0&request=GetFeature&typeNames=stat:Tatorter_2023&outputFormat=csv&propertyName=tatort,kommun,bef"
|
||||
POSTAL_CODES = "https://download.geonames.org/export/zip/SE.zip"
|
||||
NVDB = "https://api.trafikinfo.trafikverket.se/v2/data.json"
|
||||
NVDB_PAGE = 50000
|
||||
TIMEZONE = "Europe/Stockholm"
|
||||
# This script's reading of the "exceptions in big cities" in docs/research/research-geo-se.md, Postal codes → Structure.
|
||||
ONE_POSITION = {"Stockholm", "Göteborg", "Malmö"}
|
||||
# A tätort's floor: "minst 200 invånare", docs/research/research-geo-se.md, Lantmäteriet → Ortnamn.
|
||||
UNMATCHED_POPULATION = 200
|
||||
# Not a street: a road number, a ferry route, a cycle path's or road association's label, a placeholder, or two names
|
||||
# or a description in one.
|
||||
NOT_A_STREET = re.compile(r"^(Europaväg|Länsväg|Riksväg) \d|^Färjeled |\b(GC|SMF|VSF)\b|^NN$|,")
|
||||
PARTICLES = {"af", "de", "von"}
|
||||
|
||||
|
||||
def scb_codes(cache):
|
||||
regions, municipalities = {}, {}
|
||||
for cells in xlsx.rows(source.fetch(CODES, cache, "kommunlankod.xlsx", magic=b"PK")):
|
||||
if len(cells) < 2 or not re.fullmatch(r"\d{2}|\d{4}", cells[0]):
|
||||
continue
|
||||
(regions if len(cells[0]) == 2 else municipalities)[cells[0]] = cells[1].strip()
|
||||
return regions, municipalities
|
||||
|
||||
|
||||
def scb_population(cache):
|
||||
body = json.dumps(POPULATION_QUERY).encode()
|
||||
data = source.fetch(POPULATION, cache, "befolkning.json", data=body, headers={"Content-Type": "application/json"})
|
||||
return {row["key"][0]: row["values"][0] for row in json.loads(data.decode("utf-8-sig"))["data"]}
|
||||
|
||||
|
||||
def scb_tatorter(cache):
|
||||
text = source.fetch(TATORTER, cache, "tatorter.csv").decode("utf-8")
|
||||
by_name = collections.defaultdict(list)
|
||||
for r in csv.DictReader(io.StringIO(text)):
|
||||
by_name[r["tatort"]].append((r["kommun"], int(r["bef"])))
|
||||
return by_name
|
||||
|
||||
|
||||
def geonames(cache):
|
||||
z = zipfile.ZipFile(io.BytesIO(source.fetch(POSTAL_CODES, cache, "SE.zip", magic=b"PK")))
|
||||
rows = []
|
||||
for line in z.read("SE.txt").decode("utf-8").splitlines():
|
||||
f = line.split("\t")
|
||||
lat, lon = (float(f[9]), float(f[10])) if f[9] and f[10] else (None, None)
|
||||
rows.append({"code": f[1], "locality": f[2], "municipality": f[6], "lat": lat, "lon": lon})
|
||||
return rows
|
||||
|
||||
|
||||
def nvdb_segments(cache, key):
|
||||
path = cache / "nvdb-gatunamn.tsv"
|
||||
if not path.exists():
|
||||
with open(path.with_suffix(".part"), "w", encoding="utf-8") as out:
|
||||
change = "0"
|
||||
while True:
|
||||
query = (
|
||||
f'<REQUEST><LOGIN authenticationkey="{key}"/>'
|
||||
f'<QUERY objecttype="Gatunamn" namespace="vägdata.nvdb_dk_o" schemaversion="1.0" limit="{NVDB_PAGE}" changeid="{change}">'
|
||||
"<FILTER><EQ name=\"Deleted\" value=\"false\"/></FILTER>"
|
||||
"<INCLUDE>Namn</INCLUDE><INCLUDE>Geometry.WKT-WGS84-3D</INCLUDE></QUERY></REQUEST>"
|
||||
)
|
||||
req = urllib.request.Request(NVDB, data=query.encode(), headers={"Content-Type": "text/xml"})
|
||||
with urllib.request.urlopen(req, timeout=600) as r:
|
||||
result = json.load(r)["RESPONSE"]["RESULT"][0]
|
||||
rows = result.get("Gatunamn", [])
|
||||
for row in rows:
|
||||
m = re.match(r"LINESTRING Z \(([-\d.]+) ([-\d.]+) ", row.get("Geometry", {}).get("WKT-WGS84-3D", ""))
|
||||
name = " ".join(row.get("Namn", "").split())
|
||||
if m and name:
|
||||
out.write(f"{name}\t{m.group(1)}\t{m.group(2)}\n")
|
||||
change = result["INFO"]["LASTCHANGEID"]
|
||||
if len(rows) < NVDB_PAGE:
|
||||
break
|
||||
path.with_suffix(".part").rename(path)
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
name, lon, lat = line.split("\t")
|
||||
yield name, float(lat), float(lon)
|
||||
|
||||
|
||||
def street_delivery(name, codes):
|
||||
"""The codes delivered to a street. A postort's own prefix is three digits where one three-digit prefix starts at
|
||||
least half its codes, else two. Outside ONE_POSITION's cities, the digit after that prefix is 0 or 1 for boxes and 8
|
||||
for reply mail; inside them, a second digit 0 marks boxes. The box and reply-mail digits are per
|
||||
docs/research/research-geo-se.md, Postal codes → Structure."""
|
||||
if name in ONE_POSITION:
|
||||
return [c for c in codes if c[1] != "0"]
|
||||
largest = collections.Counter(c[:3] for c in codes).most_common(1)[0][1]
|
||||
length = 3 if largest * 2 >= len(codes) else 2
|
||||
return [c for c in codes if c[length] not in "018"]
|
||||
|
||||
|
||||
def municipality_of(name, rows, tatorter, municipalities):
|
||||
named = [code for code, n in municipalities.items() if n == name]
|
||||
if named:
|
||||
return named[0], "kommun"
|
||||
voted = collections.Counter(r["municipality"] for r in rows if r["municipality"] in municipalities)
|
||||
matches = tatorter.get(name, [])
|
||||
if matches:
|
||||
in_vote = [m for m in matches if voted and m[0] == voted.most_common(1)[0][0]]
|
||||
return max(in_vote or matches, key=lambda m: m[1])[0], "tatort"
|
||||
if voted:
|
||||
return voted.most_common(1)[0][0], "codes"
|
||||
return None, "unplaced"
|
||||
|
||||
|
||||
def population_of(name, municipality, tatorter, municipalities, population):
|
||||
matches = [m for m in tatorter.get(name, []) if m[0] == municipality]
|
||||
if matches:
|
||||
return str(max(m[1] for m in matches))
|
||||
return population[municipality] if municipalities[municipality] == name else str(UNMATCHED_POPULATION)
|
||||
|
||||
|
||||
def well_cased(name):
|
||||
"""Whether each word of name is capitalised: GeoNames also spells some postorter, such as STOCKHOLM and örebro, in
|
||||
capitals or lower case."""
|
||||
return all(part[:1].isupper() and (len(part) == 1 or not part.isupper()) for part in re.split(r"[ -]", name))
|
||||
|
||||
|
||||
def localities(codes, tatorter, municipalities, population):
|
||||
"""Each postort with its municipality, weight, centroid and street-delivery codes."""
|
||||
by_locality = collections.defaultdict(list)
|
||||
for r in codes:
|
||||
by_locality[r["locality"]].append(r)
|
||||
out, how = {}, collections.Counter()
|
||||
for name, rows in by_locality.items():
|
||||
municipality, method = municipality_of(name, rows, tatorter, municipalities)
|
||||
how[method] += 1
|
||||
kept = street_delivery(name, [r["code"].replace(" ", "") for r in rows])
|
||||
point = geo.centroid(rows)
|
||||
if municipality is None or not kept or point is None or not well_cased(name):
|
||||
continue
|
||||
lat, lon = point
|
||||
out[name] = {"name": name, "municipality": municipality, "population": population_of(name, municipality, tatorter, municipalities, population), "lat": f"{lat:.4f}", "lon": f"{lon:.4f}", "codes": kept}
|
||||
print(f"municipality by {dict(how)}; {len(by_locality) - len(out)} postorter dropped before streets", file=sys.stderr)
|
||||
return out
|
||||
|
||||
|
||||
def street_name(name):
|
||||
# A word of four capitals or fewer is an acronym or a Roman numeral: IOGT-vägen, Karl XII gatan.
|
||||
opening = name[0].isupper() or name.split()[0] in PARTICLES
|
||||
return opening and not NOT_A_STREET.search(name) and not any(len(w) >= 5 and w.isupper() for w in name.split())
|
||||
|
||||
|
||||
def streets(segments, codes, localities, share, minimum):
|
||||
"""The names with most segments per locality, each segment at its nearest code centroid."""
|
||||
nearest = geo.Nearest((r["lat"], r["lon"], r["locality"]) for r in codes if r["lat"] is not None and r["locality"] in localities)
|
||||
count = collections.Counter()
|
||||
for name, lat, lon in segments:
|
||||
if street_name(name):
|
||||
count[(nearest.find(lat, lon), name)] += 1
|
||||
return geo.top_streets(count, share, minimum, "segments")
|
||||
|
||||
|
||||
def main():
|
||||
a = geo.parser(__doc__, "SE", 1.0, key_file={"help": "file holding the Trafikverket API key; TRAFIKVERKET_API_KEY otherwise"}).parse_args()
|
||||
key = Path(a.key_file).read_text().strip() if a.key_file else os.environ.get("TRAFIKVERKET_API_KEY")
|
||||
if not key:
|
||||
sys.exit("set TRAFIKVERKET_API_KEY or pass --key-file")
|
||||
cache, out = geo.directories(a)
|
||||
|
||||
region_names, municipality_names = scb_codes(cache)
|
||||
population = scb_population(cache)
|
||||
codes = geonames(cache)
|
||||
places = localities(codes, scb_tatorter(cache), municipality_names, population)
|
||||
named = streets(nvdb_segments(cache, key), codes, places, a.street_share, a.min_streets)
|
||||
regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in region_names.items()}
|
||||
municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipality_names.items()}
|
||||
kept, dropped = geo.fence(regions, municipalities, places, named, {k: l["codes"] for k, l in places.items()})
|
||||
geo.log_dropped_localities(dropped["locality"])
|
||||
if dropped["municipality"] or dropped["region"]:
|
||||
sys.exit(f"municipalities without a locality: {sorted(dropped['municipality'])}; regions without a municipality: {sorted(dropped['region'])}")
|
||||
places = kept["locality"]
|
||||
|
||||
tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
|
||||
tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(kept["municipality"].items())])
|
||||
tsv.write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
|
||||
tsv.write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in places.values() for c in l["codes"]), key=lambda r: r["code"]))
|
||||
tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(places) for s in named[locality]])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,237 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild data/geo/US/*.tsv from the Census Bureau's Gazetteer, population estimates, ZCTA relationships and TIGER/Line files (public domain), and Wikidata's state capitals (CC0).
|
||||
|
||||
data-import/geo-us.py [--cache DIR] [--min-streets N] [--place-share F] [--street-share F] [--out DIR]
|
||||
"""
|
||||
import collections
|
||||
import concurrent.futures
|
||||
import csv
|
||||
import io
|
||||
import math
|
||||
import random
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
import urllib.parse
|
||||
import zipfile
|
||||
|
||||
import geo
|
||||
import source
|
||||
import tsv
|
||||
|
||||
GAZETTEER = "https://www2.census.gov/geo/docs/maps-data/data/gazetteer/2026_Gazetteer/2026_Gaz_{}_national.zip"
|
||||
POPULATION = "https://www2.census.gov/programs-surveys/popest/datasets/2020-2025/{}"
|
||||
STATES = POPULATION.format("state/totals/NST-EST2025-ALLDATA.csv")
|
||||
COUNTIES = POPULATION.format("counties/totals/co-est2025-alldata.csv")
|
||||
PLACES = POPULATION.format("cities/totals/sub-est2025.csv")
|
||||
ZCTA_PLACE = "https://www2.census.gov/geo/docs/maps-data/data/rel2020/zcta520/tab20_zcta520_place20_natl.txt"
|
||||
TIGER = "https://www2.census.gov/geo/tiger/TIGER2025/{0}/tl_2025_{1}_{2}.zip"
|
||||
ESTIMATE = "POPESTIMATE2025"
|
||||
CDP = "57"
|
||||
# Each state's capital from Wikidata, with the capital's GNIS ids, which the Gazetteer carries as ANSICODE.
|
||||
CAPITALS = "https://query.wikidata.org/sparql?" + urllib.parse.urlencode({"query": """
|
||||
SELECT ?iso ?capitalLabel ?gnis WHERE {
|
||||
?state wdt:P31 wd:Q35657; wdt:P300 ?iso; wdt:P36 ?capital.
|
||||
OPTIONAL { ?capital wdt:P590 ?gnis. }
|
||||
SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
|
||||
}"""})
|
||||
SHIPPED_POPULATION = 100000
|
||||
HIGHWAY = re.compile(r"\b(I- |Hwy |Highway |Loop |Rte |Route |Rd )\d")
|
||||
SUFFIX = re.compile(r" (city and borough|city|town|village|borough|municipality|comunidad|zona urbana|metropolitan government|metro government|consolidated government|unified government|urban county|corporation|plantation)( \(balance\))?$")
|
||||
# Places whose Census name is not the postal city an address carries.
|
||||
NAMES = {"0665042": "Ventura", "1303440": "Athens", "1304204": "Augusta", "1349008": "Macon", "1608830": "Boise", "2146027": "Lexington", "2148006": "Louisville", "3011397": "Butte", "4732742": "Hartsville", "4744382": "Lynchburg", "4752006": "Nashville"}
|
||||
TIMEZONES = {
|
||||
"AK": "America/Anchorage", "AL": "America/Chicago", "AR": "America/Chicago", "AZ": "America/Phoenix",
|
||||
"CA": "America/Los_Angeles", "CO": "America/Denver", "CT": "America/New_York", "DC": "America/New_York",
|
||||
"DE": "America/New_York", "FL": "America/New_York", "GA": "America/New_York", "HI": "Pacific/Honolulu",
|
||||
"IA": "America/Chicago", "ID": "America/Boise", "IL": "America/Chicago", "IN": "America/Indiana/Indianapolis",
|
||||
"KS": "America/Chicago", "KY": "America/New_York", "LA": "America/Chicago", "MA": "America/New_York",
|
||||
"MD": "America/New_York", "ME": "America/New_York", "MI": "America/Detroit", "MN": "America/Chicago",
|
||||
"MO": "America/Chicago", "MS": "America/Chicago", "MT": "America/Denver", "NC": "America/New_York",
|
||||
"ND": "America/Chicago", "NE": "America/Chicago", "NH": "America/New_York", "NJ": "America/New_York",
|
||||
"NM": "America/Denver", "NV": "America/Los_Angeles", "NY": "America/New_York", "OH": "America/New_York",
|
||||
"OK": "America/Chicago", "OR": "America/Los_Angeles", "PA": "America/New_York", "PR": "America/Puerto_Rico",
|
||||
"RI": "America/New_York", "SC": "America/New_York", "SD": "America/Chicago", "TN": "America/Chicago",
|
||||
"TX": "America/Chicago", "UT": "America/Denver", "VA": "America/New_York", "VT": "America/New_York",
|
||||
"WA": "America/Los_Angeles", "WI": "America/Chicago", "WV": "America/New_York", "WY": "America/Denver",
|
||||
}
|
||||
|
||||
|
||||
def text(data):
|
||||
try:
|
||||
return data.decode("utf-8-sig")
|
||||
except UnicodeDecodeError:
|
||||
return data.decode("latin-1")
|
||||
|
||||
|
||||
def gazetteer(cache, kind):
|
||||
z = zipfile.ZipFile(io.BytesIO(source.fetch(GAZETTEER.format(kind), cache, f"gaz_{kind}.zip", magic=b"PK")))
|
||||
rows = text(z.read(z.namelist()[0])).splitlines()
|
||||
header = [h.strip() for h in rows[0].split("|")]
|
||||
return [dict(zip(header, (c.strip() for c in row.split("|")))) for row in rows[1:]]
|
||||
|
||||
|
||||
def csv_rows(cache, url, name):
|
||||
return list(csv.DictReader(io.StringIO(text(source.fetch(url, cache, name)))))
|
||||
|
||||
|
||||
def dbf_rows(data, wanted):
|
||||
"""The records of a dBASE file, the wanted fields only."""
|
||||
count, header_len, record_len = struct.unpack("<xxxxIHH", data[:12])
|
||||
fields, pos = [], 32
|
||||
while data[pos] != 0x0D:
|
||||
name = data[pos:pos + 11].split(b"\0")[0].decode()
|
||||
fields.append((name, data[pos + 16]))
|
||||
pos += 32
|
||||
pos = header_len
|
||||
for _ in range(count):
|
||||
record = data[pos:pos + record_len]
|
||||
pos += record_len
|
||||
if record[:1] == b"*":
|
||||
continue
|
||||
row, at = {}, 1
|
||||
for name, length in fields:
|
||||
if name in wanted:
|
||||
row[name] = record[at:at + length].decode("utf-8", "replace").strip()
|
||||
at += length
|
||||
yield row
|
||||
|
||||
|
||||
def tiger_zip(cache, kind, county):
|
||||
return source.fetch(TIGER.format(kind.upper(), county, kind), cache, f"tl_{county}_{kind}.zip", magic=b"PK")
|
||||
|
||||
|
||||
def tiger(cache, kind, county, wanted):
|
||||
z = zipfile.ZipFile(io.BytesIO(tiger_zip(cache, kind, county)))
|
||||
return dbf_rows(z.read(f"tl_2025_{county}_{kind}.dbf"), wanted)
|
||||
|
||||
|
||||
def place_name(geoid, name):
|
||||
if geoid in NAMES:
|
||||
return NAMES[geoid]
|
||||
stripped = SUFFIX.sub("", name)
|
||||
if stripped == name:
|
||||
print(f"{geoid}: no suffix stripped from {name!r}", file=sys.stderr)
|
||||
return stripped
|
||||
|
||||
|
||||
def localities(cache, counties):
|
||||
"""Each incorporated place with its county, population and centroid."""
|
||||
place_population, county_part = {}, collections.defaultdict(list)
|
||||
for r in csv_rows(cache, PLACES, "sub-est2025.csv"):
|
||||
if r["SUMLEV"] == "162":
|
||||
place_population[r["STATE"] + r["PLACE"]] = int(r[ESTIMATE])
|
||||
elif r["SUMLEV"] == "157":
|
||||
county_part[r["STATE"] + r["PLACE"]].append((int(r[ESTIMATE]), r["STATE"] + r["COUNTY"]))
|
||||
out = {}
|
||||
for r in gazetteer(cache, "place"):
|
||||
geoid, population = r["GEOID"], place_population.get(r["GEOID"], 0)
|
||||
if r["FUNCSTAT"] not in ("A", "B", "F", "N") or r["LSAD"] == CDP or not county_part.get(geoid):
|
||||
continue
|
||||
county = max(county_part[geoid])[1]
|
||||
if county not in counties:
|
||||
print(f"{geoid} {r['NAME']}: county {county} unknown, dropped", file=sys.stderr)
|
||||
continue
|
||||
out[geoid] = {"code": geoid, "gnis": int(r["ANSICODE"]), "name": place_name(geoid, r["NAME"]), "municipality": county, "population": population, "lat": r["INTPTLAT"], "lon": r["INTPTLONG"]}
|
||||
return out
|
||||
|
||||
|
||||
def capitals(cache, places, counties):
|
||||
"""The codes of the places that are a state's capital, matched by any of its GNIS ids, else by a name no other place in the state carries."""
|
||||
by_gnis = {p["gnis"]: code for code, p in places.items()}
|
||||
named = collections.defaultdict(list)
|
||||
for code, p in places.items():
|
||||
named[(counties[p["municipality"]]["region"], p["name"])].append(code)
|
||||
by_name = {key: codes[0] for key, codes in named.items() if len(codes) == 1}
|
||||
gnis_of = collections.defaultdict(set)
|
||||
for r in csv.DictReader(io.StringIO(text(source.fetch(CAPITALS, cache, "capitals.csv", magic=b"iso,", headers={"Accept": "text/csv"})))):
|
||||
gnis_of[(r["iso"].removeprefix("US-"), r["capitalLabel"])].add(int(r["gnis"] or 0))
|
||||
out = set()
|
||||
for (state, name), ids in sorted(gnis_of.items()):
|
||||
code = next((by_gnis[i] for i in sorted(ids) if i in by_gnis), None) or by_name.get((state, name))
|
||||
if code:
|
||||
out.add(code)
|
||||
else:
|
||||
print(f"{state} {name}: capital matches no incorporated place by GNIS id or by a name unique in its state", file=sys.stderr)
|
||||
return out
|
||||
|
||||
|
||||
def drawn(places, share):
|
||||
"""The share of the places with people, each drawn with odds by its population: Efraimidis and Spirakis's weighted
|
||||
sampling without replacement. Each place's random number is seeded by its code, so a place added to or dropped from
|
||||
the sources moves only the places near the cut."""
|
||||
peopled = [code for code, p in places.items() if p["population"] > 0]
|
||||
key = {code: math.log(1 - random.Random(code).random()) / places[code]["population"] for code in peopled}
|
||||
return {code: places[code] for code in sorted(peopled, key=lambda c: (key[c], c), reverse=True)[:round(len(peopled) * share)]}
|
||||
|
||||
|
||||
def zcta_localities(cache, localities):
|
||||
"""Each ZCTA mapped to the incorporated place holding most of its land inside incorporated places, where localities holds that place."""
|
||||
parts = {}
|
||||
for r in csv.DictReader(io.StringIO(text(source.fetch(ZCTA_PLACE, cache, "zcta-place.txt"))), delimiter="|"):
|
||||
if r["GEOID_ZCTA5_20"] and r["GEOID_PLACE_20"] and not r["NAMELSAD_PLACE_20"].endswith(" CDP"):
|
||||
parts.setdefault(r["GEOID_ZCTA5_20"], []).append((int(r["AREALAND_PART"]), r["GEOID_PLACE_20"]))
|
||||
largest = {zcta: max(p)[1] for zcta, p in parts.items()}
|
||||
return {zcta: place for zcta, place in largest.items() if place in localities}
|
||||
|
||||
|
||||
def streets(cache, counties, locality_of_zcta, share, minimum):
|
||||
"""The names with most TIGER address ranges per place, and the address ranges per ZCTA."""
|
||||
with concurrent.futures.ThreadPoolExecutor(3) as pool:
|
||||
list(pool.map(lambda c: (tiger_zip(cache, "addr", c), tiger_zip(cache, "featnames", c)), counties))
|
||||
addresses, count = collections.Counter(), collections.Counter()
|
||||
for county in counties:
|
||||
zips = collections.defaultdict(set)
|
||||
for r in tiger(cache, "addr", county, {"TLID", "ZIP"}):
|
||||
if r["ZIP"] in locality_of_zcta:
|
||||
zips[r["TLID"]].add(r["ZIP"])
|
||||
addresses[r["ZIP"]] += 1
|
||||
for r in tiger(cache, "featnames", county, {"TLID", "FULLNAME", "PAFLAG"}):
|
||||
if r["PAFLAG"] == "P" and r["FULLNAME"] and not HIGHWAY.search(r["FULLNAME"]):
|
||||
for z in zips.get(r["TLID"], ()):
|
||||
count[(locality_of_zcta[z], r["FULLNAME"])] += 1
|
||||
return geo.top_streets(count, share, minimum, "addresses"), addresses
|
||||
|
||||
|
||||
def main():
|
||||
a = geo.parser(__doc__, "US", 0.25, place_share={"type": float, "default": 0.1}).parse_args()
|
||||
cache, out = geo.directories(a)
|
||||
|
||||
state_population = {r["STATE"]: r[ESTIMATE] for r in csv_rows(cache, STATES, "nst-est2025.csv") if r["SUMLEV"] == "040"}
|
||||
regions = {r["USPS"]: {"abbr": r["USPS"], "code": r["GEOID"], "name": r["NAME"], "population": state_population[r["GEOID"]], "timezone": TIMEZONES[r["USPS"]]} for r in gazetteer(cache, "state")}
|
||||
county_population = {r["STATE"] + r["COUNTY"]: r[ESTIMATE] for r in csv_rows(cache, COUNTIES, "co-est2025.csv") if r["SUMLEV"] == "050"}
|
||||
counties, unestimated = {}, collections.Counter()
|
||||
for r in gazetteer(cache, "counties"):
|
||||
if r["GEOID"] not in county_population:
|
||||
unestimated[r["USPS"]] += 1
|
||||
continue
|
||||
counties[r["GEOID"]] = {"code": r["GEOID"], "name": r["NAME"], "region": r["USPS"], "population": county_population[r["GEOID"]]}
|
||||
print(f"counties without a population estimate, dropped: {dict(unestimated)}", file=sys.stderr)
|
||||
|
||||
places = localities(cache, counties)
|
||||
owner_of_zcta = zcta_localities(cache, places)
|
||||
capital_codes = capitals(cache, places, counties)
|
||||
owners = {p: places[p] for p in set(owner_of_zcta.values())}
|
||||
for code in sorted(capital_codes - set(owners)):
|
||||
print(f"{code} {places[code]['name']}: capital owns no ZCTA, dropped", file=sys.stderr)
|
||||
places = drawn(owners, a.place_share) | {code: p for code, p in owners.items() if code in capital_codes or p["population"] >= SHIPPED_POPULATION}
|
||||
locality_of_zcta = {zcta: p for zcta, p in owner_of_zcta.items() if p in places}
|
||||
named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.street_share, a.min_streets)
|
||||
zctas = collections.defaultdict(list)
|
||||
for zcta, locality in sorted(locality_of_zcta.items()):
|
||||
if addresses[zcta]:
|
||||
zctas[locality].append(zcta)
|
||||
kept, dropped = geo.fence(regions, counties, places, named, zctas)
|
||||
geo.log_dropped_localities(dropped["locality"])
|
||||
print(f"counties without a place, dropped: {len(dropped['municipality'])}; states without a county, dropped: {sorted(dropped['region'])}", file=sys.stderr)
|
||||
places = kept["locality"]
|
||||
|
||||
tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
|
||||
tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(kept["municipality"].items())])
|
||||
tsv.write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
|
||||
tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], sorted(({"code": z, "locality": l, "addresses": addresses[z]} for l in places for z in zctas[l]), key=lambda r: r["code"]))
|
||||
tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(places) for s in named[locality]])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,98 +0,0 @@
|
||||
"""The steps a country's geo import may share with another's."""
|
||||
import argparse
|
||||
import collections
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
DATA = Path(__file__).resolve().parent.parent / "data" / "geo"
|
||||
CACHE = Path(__file__).resolve().parent / "cache"
|
||||
|
||||
|
||||
def parser(doc, country, street_share, **own):
|
||||
"""A parser for the options every country's import takes, plus the script's own: each keyword names one option, with
|
||||
"_" standing for "-"."""
|
||||
options = {"cache": {"default": str(CACHE)}, "out": {"default": str(DATA / country)}, "min_streets": {"type": int, "default": 10}, "street_share": {"type": float, "default": street_share}, **own}
|
||||
p = argparse.ArgumentParser(description=doc.splitlines()[0])
|
||||
for name in sorted(options):
|
||||
p.add_argument("--" + name.replace("_", "-"), **options[name])
|
||||
return p
|
||||
|
||||
|
||||
def directories(args):
|
||||
"""The cache and out directories of the parsed options, both created."""
|
||||
cache, out = Path(args.cache), Path(args.out)
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
return cache, out
|
||||
|
||||
|
||||
def centroid(rows):
|
||||
"""The mean lat and lon of the rows with a point, or None where no row has one."""
|
||||
points = [r for r in rows if r["lat"] is not None]
|
||||
if not points:
|
||||
return None
|
||||
return sum(r["lat"] for r in points) / len(points), sum(r["lon"] for r in points) / len(points)
|
||||
|
||||
|
||||
class Nearest:
|
||||
"""Nearest point by an equirectangular distance, over a degree grid."""
|
||||
|
||||
def __init__(self, points, cell=0.05):
|
||||
self.cell = cell
|
||||
self.grid = collections.defaultdict(list)
|
||||
for lat, lon, value in points:
|
||||
self.grid[(int(lat // cell), int(lon // cell))].append((lat, lon, value))
|
||||
|
||||
def find(self, lat, lon):
|
||||
ci, cj = int(lat // self.cell), int(lon // self.cell)
|
||||
best, best_d = None, math.inf
|
||||
ring = 0
|
||||
while ring < 400:
|
||||
for i in range(ci - ring, ci + ring + 1):
|
||||
for j in range(cj - ring, cj + ring + 1):
|
||||
if max(abs(i - ci), abs(j - cj)) != ring:
|
||||
continue
|
||||
for plat, plon, value in self.grid.get((i, j), ()):
|
||||
d = (plat - lat) ** 2 + ((plon - lon) * math.cos(math.radians(lat))) ** 2
|
||||
if d < best_d:
|
||||
best, best_d = value, d
|
||||
if best is not None and math.sqrt(best_d) < ring * self.cell * math.cos(math.radians(lat)):
|
||||
return best
|
||||
ring += 1
|
||||
return best
|
||||
|
||||
|
||||
def top_streets(count, share, minimum, column):
|
||||
"""count maps (locality, street name) to a count. Returns, per locality, rows for its most-counted names, share of
|
||||
them rounded up and at least minimum, ties broken by name; each row holds its count under column."""
|
||||
of = collections.defaultdict(list)
|
||||
for (locality, name), n in count.items():
|
||||
of[locality].append((n, name))
|
||||
return {locality: [{"name": name, "locality": locality, column: n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:max(minimum, math.ceil(len(named) * share))]] for locality, named in of.items()}
|
||||
|
||||
|
||||
def fence(regions, municipalities, localities, streets, postal_codes):
|
||||
"""Meet the loader's fence that every parent row has a child row. Keeps each locality with entries in both streets
|
||||
and postal_codes, then each municipality and region left with a child.
|
||||
|
||||
regions and municipalities map a code to a row; a municipality row names its region under "region". localities,
|
||||
streets and postal_codes map a locality's key to its row, its streets and its postal codes. A locality row holds
|
||||
name, municipality, population, lat and lon, plus its country's own keys.
|
||||
|
||||
Returns kept, then dropped: each maps "locality", "municipality" and "region" to that table's rows."""
|
||||
kept = {"locality": {k: l for k, l in localities.items() if streets.get(k) and postal_codes.get(k)}}
|
||||
municipality_codes = {l["municipality"] for l in kept["locality"].values()}
|
||||
kept["municipality"] = {c: m for c, m in municipalities.items() if c in municipality_codes}
|
||||
region_codes = {m["region"] for m in kept["municipality"].values()}
|
||||
kept["region"] = {c: r for c, r in regions.items() if c in region_codes}
|
||||
every = {"locality": localities, "municipality": municipalities, "region": regions}
|
||||
dropped = {table: {k: row for k, row in rows.items() if k not in kept[table]} for table, rows in every.items()}
|
||||
return kept, dropped
|
||||
|
||||
|
||||
def log_dropped_localities(rows):
|
||||
"""Logs each locality row as dropped, by its key, plus its name where that differs."""
|
||||
for key, row in rows.items():
|
||||
label = key if key == row["name"] else f"{key} {row['name']}"
|
||||
print(f"{label}: no street or no postal code, dropped", file=sys.stderr)
|
||||
@@ -1,59 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild data/misc/httpmethod.tsv from the IANA HTTP Method Registry.
|
||||
|
||||
data-import/httpmethod.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
|
||||
|
||||
A method ships when the register cites RFC 9110 section 9.3 or RFC 5789 for it, which
|
||||
is the eight HTTP defines and PATCH; the register's other entries do not. The
|
||||
register's `yes` and `no` ship as `true` and `false`, which a consumer's boolean reads.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import source
|
||||
import tsv
|
||||
|
||||
SOURCE = "https://www.iana.org/assignments/http-methods/methods.csv"
|
||||
OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "httpmethod.tsv"
|
||||
CACHE = Path(__file__).resolve().parent / "cache"
|
||||
COLUMNS = ["idempotent", "method", "safe"]
|
||||
CORE = re.compile(r"\[RFC9110, Section 9\.3\.|\[RFC5789, Section 2\]")
|
||||
BOOLEAN = {"yes": "true", "no": "false"}
|
||||
SELECTOR = '[]{}"|' # mirrors inSelector in internal/grammar/path.go
|
||||
EXPECTED = 9
|
||||
|
||||
|
||||
def rows(text):
|
||||
for r in csv.DictReader(io.StringIO(text)):
|
||||
method = (r["Method Name"] or "").strip()
|
||||
if not CORE.search(r["Reference"] or ""):
|
||||
continue
|
||||
if set(method) & set(SELECTOR):
|
||||
sys.exit(f"{method}: a method holding one of {SELECTOR} is one a selector cannot spell")
|
||||
answers = {}
|
||||
for column in ("Safe", "Idempotent"):
|
||||
answer = (r[column] or "").strip()
|
||||
if answer not in BOOLEAN:
|
||||
sys.exit(f"{method}: {column} {answer!r} is neither yes nor no")
|
||||
answers[column] = BOOLEAN[answer]
|
||||
yield {"idempotent": answers["Idempotent"], "method": method, "safe": answers["Safe"]}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("--cache", default=str(CACHE))
|
||||
p.add_argument("--source", default=SOURCE)
|
||||
p.add_argument("--out", default=str(OUT))
|
||||
a = p.parse_args()
|
||||
table = sorted(rows(source.fetch(a.source, a.cache, "http-methods.csv").decode("utf-8")), key=lambda r: r["method"])
|
||||
if len(table) != EXPECTED:
|
||||
sys.exit(f"{len(table)} methods cite RFC 9110 section 9.3 or RFC 5789, not {EXPECTED}; the register's Reference column has moved")
|
||||
tsv.write(a.out, COLUMNS, table)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,40 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild data/misc/httpstatus.tsv from the IANA HTTP Status Code Registry.
|
||||
|
||||
data-import/httpstatus.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
|
||||
|
||||
What ships is the codes in use: a range, an unassigned code and a qualified reason are all skipped.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
from pathlib import Path
|
||||
|
||||
import source
|
||||
import tsv
|
||||
|
||||
SOURCE = "https://www.iana.org/assignments/http-status-codes/http-status-codes-1.csv"
|
||||
OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "httpstatus.tsv"
|
||||
CACHE = Path(__file__).resolve().parent / "cache"
|
||||
COLUMNS = ["code", "reason"]
|
||||
|
||||
|
||||
def rows(text):
|
||||
for r in csv.DictReader(io.StringIO(text)):
|
||||
code, reason = r["Value"].strip(), r["Description"].strip()
|
||||
if code.isdigit() and reason and "(" not in reason and reason != "Unassigned":
|
||||
yield {"code": code, "reason": reason}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("--cache", default=str(CACHE))
|
||||
p.add_argument("--source", default=SOURCE)
|
||||
p.add_argument("--out", default=str(OUT))
|
||||
a = p.parse_args()
|
||||
table = rows(source.fetch(a.source, a.cache, "http-status-codes-1.csv").decode("utf-8"))
|
||||
tsv.write(a.out, COLUMNS, sorted(table, key=lambda r: int(r["code"])))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,45 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild data/misc/language.tsv from datasets/language-codes (PDDL), the LoC ISO 639-2 register.
|
||||
|
||||
data-import/language.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
|
||||
|
||||
Only the 639-2 entries carrying a 639-1 code ship, since the alpha-2 code is the key.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import source
|
||||
import tsv
|
||||
|
||||
SOURCE = "https://raw.githubusercontent.com/datasets/language-codes/main/data/language-codes-full.csv"
|
||||
OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "language.tsv"
|
||||
CACHE = Path(__file__).resolve().parent / "cache"
|
||||
COLUMNS = ["code", "code3", "name"]
|
||||
|
||||
|
||||
def name(english):
|
||||
"""The register lists every synonym; the first, without its qualifier, is the one a selector spells."""
|
||||
return re.sub(r"\s*\([^)]*\)", "", english.split(";")[0]).strip()
|
||||
|
||||
|
||||
def rows(text):
|
||||
for r in csv.DictReader(io.StringIO(text)):
|
||||
if r["alpha2"]:
|
||||
yield {"code": r["alpha2"], "code3": r["alpha3-t"] or r["alpha3-b"], "name": name(r["English"])}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("--cache", default=str(CACHE))
|
||||
p.add_argument("--source", default=SOURCE)
|
||||
p.add_argument("--out", default=str(OUT))
|
||||
a = p.parse_args()
|
||||
table = rows(source.fetch(a.source, a.cache, "language-codes-full.csv").decode("utf-8"))
|
||||
tsv.write(a.out, COLUMNS, sorted(table, key=lambda r: r["code"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,68 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild data/misc/loglevel.tsv from RFC 5424 and POSIX <syslog.h>.
|
||||
|
||||
data-import/loglevel.py [--source URL_OR_FILE] [--posix URL_OR_FILE] [--cache DIR] [--out FILE]
|
||||
|
||||
Every severity in Table 2 ships, paired to POSIX's keywords by the order each source
|
||||
lists them, which holds only because a keyword prefixes the severity RFC 5424 numbers
|
||||
alike.
|
||||
"""
|
||||
import argparse
|
||||
import html
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import source
|
||||
import tsv
|
||||
|
||||
SOURCE = "https://www.rfc-editor.org/rfc/rfc5424.txt"
|
||||
POSIX = "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/syslog.h.html"
|
||||
OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "loglevel.tsv"
|
||||
CACHE = Path(__file__).resolve().parent / "cache"
|
||||
COLUMNS = ["code", "keyword", "severity"]
|
||||
TABLE = re.compile(r"Table 1\.\s+Syslog Message Facilities(.*?)Table 2\.\s+Syslog Message Severities", re.S)
|
||||
ROW = re.compile(r"^ +(\d) +([A-Z][a-z]+): ", re.M)
|
||||
SECTION = re.compile(r"severity level portion of the.*?macros:(.*?)The following shall be declared as functions", re.S)
|
||||
MACRO = re.compile(r"\bLOG_([A-Z]+)\b")
|
||||
EXPECTED = 8
|
||||
|
||||
|
||||
def severities(text):
|
||||
table = TABLE.search(text)
|
||||
if not table:
|
||||
sys.exit("the RFC holds no text between the facility and the severity caption; its tables have moved")
|
||||
return ROW.findall(table.group(1))
|
||||
|
||||
|
||||
def keywords(page):
|
||||
section = SECTION.search(html.unescape(re.sub(r"<[^>]+>", "", page)))
|
||||
if not section:
|
||||
sys.exit("POSIX spells no severity list between its own caption and the function declarations; the page has moved")
|
||||
return [m.lower() for m in MACRO.findall(section.group(1))]
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("--cache", default=str(CACHE))
|
||||
p.add_argument("--source", default=SOURCE)
|
||||
p.add_argument("--posix", default=POSIX)
|
||||
p.add_argument("--out", default=str(OUT))
|
||||
a = p.parse_args()
|
||||
table = sorted(severities(source.fetch(a.source, a.cache, "rfc5424.txt").decode("utf-8")))
|
||||
codes = [code for code, _ in table]
|
||||
if codes != [str(i) for i in range(EXPECTED)]:
|
||||
sys.exit(f"the severity table numbers {codes}, not 0 through {EXPECTED - 1}")
|
||||
words = keywords(source.fetch(a.posix, a.cache, "posix-syslog.h.html").decode("utf-8"))
|
||||
if len(words) != EXPECTED:
|
||||
sys.exit(f"POSIX lists {len(words)} severity macros, not {EXPECTED}: {words}")
|
||||
rows = []
|
||||
for (code, severity), keyword in zip(table, words):
|
||||
if not severity.lower().startswith(keyword):
|
||||
sys.exit(f"severity {code} is {severity} in the RFC and {keyword} in POSIX, which is no prefix of it")
|
||||
rows.append({"code": code, "keyword": keyword, "severity": severity})
|
||||
tsv.write(a.out, COLUMNS, rows)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,39 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild data/misc/mimetype.tsv from mime-db (MIT), the IANA media type registry with extensions.
|
||||
|
||||
data-import/mimetype.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
|
||||
|
||||
Only types mime-db records as IANA-registered, with a filename extension, ship.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import source
|
||||
import tsv
|
||||
|
||||
SOURCE = "https://raw.githubusercontent.com/jshttp/mime-db/master/db.json"
|
||||
OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "mimetype.tsv"
|
||||
CACHE = Path(__file__).resolve().parent / "cache"
|
||||
COLUMNS = ["ext", "type"]
|
||||
FIXUPS = {"application/mp4": "mp4s", "audio/mpeg": "mp3", "audio/ogg": "ogg", "video/quicktime": "mov"}
|
||||
|
||||
|
||||
def rows(db):
|
||||
for mimetype, entry in db.items():
|
||||
if entry.get("source") == "iana" and entry.get("extensions"):
|
||||
yield {"ext": "." + FIXUPS.get(mimetype, entry["extensions"][0]), "type": mimetype}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("--cache", default=str(CACHE))
|
||||
p.add_argument("--source", default=SOURCE)
|
||||
p.add_argument("--out", default=str(OUT))
|
||||
a = p.parse_args()
|
||||
table = rows(json.loads(source.fetch(a.source, a.cache, "mime-db.json").decode("utf-8")))
|
||||
tsv.write(a.out, COLUMNS, sorted(table, key=lambda r: r["type"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,54 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild data/sv_SE/first-name.tsv and last-name.tsv from SCB's whole-population name counts (CC0, "Källa: SCB").
|
||||
|
||||
data-import/names-se.py [--source URL_OR_FILE] [--first N] [--last N] [--cache DIR] [--out DIR]
|
||||
|
||||
A tilltalsnamn goes under each sex that carries it, so a name both sexes carry has two rows.
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import source
|
||||
import tsv
|
||||
import xlsx
|
||||
|
||||
SOURCE = "https://www.scb.se/contentassets/9fe7dbb460994c72b835163dbc491ef9/namn-med-minst-tva-barare-31-december-2022.xlsx"
|
||||
OUT = Path(__file__).resolve().parent.parent / "data" / "sv_SE"
|
||||
CACHE = Path(__file__).resolve().parent / "cache"
|
||||
SHEETS = {"f": "Tilltalsnamn kvinnor", "m": "Tilltalsnamn män"}
|
||||
SURNAMES = "Efternamn"
|
||||
NAME = re.compile(r"^[^\W\d_]{2,}([ '-][^\W\d_]{2,})*$")
|
||||
PARTICLES = {"af", "av", "da", "de", "del", "den", "der", "di", "dos", "du", "la", "le", "van", "von"}
|
||||
|
||||
|
||||
def cased(name):
|
||||
"""SCB's uppercase name as it is written: each part capitalised, a particle before another part lowercased."""
|
||||
parts = name.lower().split(" ")
|
||||
return " ".join(w if w in PARTICLES and len(parts) > 1 else w.title() for w in parts)
|
||||
|
||||
|
||||
def counted(rows, top):
|
||||
"""The top names of a sheet by bearers, initials and single letters dropped."""
|
||||
names = [(cells[0], int(cells[1])) for cells in rows if len(cells) >= 2 and cells[1].isdigit() and NAME.match(cells[0])]
|
||||
return sorted(names, key=lambda n: (-n[1], n[0]))[:top]
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("--cache", default=str(CACHE))
|
||||
p.add_argument("--first", type=int, default=2000, help="names per sex")
|
||||
p.add_argument("--last", type=int, default=5000)
|
||||
p.add_argument("--out", default=str(OUT))
|
||||
p.add_argument("--source", default=SOURCE)
|
||||
a = p.parse_args()
|
||||
data = source.fetch(a.source, a.cache, "scb-namn-2022.xlsx", magic=b"PK")
|
||||
first = [{"name": cased(n), "sex": sex, "count": c} for sex, sheet in SHEETS.items() for n, c in counted(xlsx.rows(data, sheet), a.first)]
|
||||
last = [{"name": cased(n), "count": c} for n, c in counted(xlsx.rows(data, SURNAMES), a.last)]
|
||||
out = Path(a.out)
|
||||
tsv.write(out / "first-name.tsv", ["name", "sex", "count"], sorted(first, key=lambda r: (r["name"], r["sex"])))
|
||||
tsv.write(out / "last-name.tsv", ["name", "count"], sorted(last, key=lambda r: r["name"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,79 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild data/en_US/first-name.tsv and last-name.tsv from SSA baby names and the Census 2010 surnames (public domain).
|
||||
|
||||
data-import/names-us.py [--names URL_OR_FILE] [--surnames URL_OR_FILE] [--from-year YEAR] [--first N] [--last N] [--cache DIR] [--out DIR]
|
||||
|
||||
A given name is counted over the births of --from-year and later, under each sex it was given to, so a
|
||||
name both sexes carry has two rows. --names takes SSA's names.zip or a merged name,sex,count,year file;
|
||||
--surnames takes the Census names.zip or its Names_2010Census.csv.
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import source
|
||||
import tsv
|
||||
|
||||
NAMES = "https://raw.githubusercontent.com/hackerb9/ssa-baby-names/master/alldata.txt"
|
||||
SURNAMES = "https://www2.census.gov/topics/genealogy/2010surnames/names.zip"
|
||||
OUT = Path(__file__).resolve().parent.parent / "data" / "en_US"
|
||||
CACHE = Path(__file__).resolve().parent / "cache"
|
||||
MC = re.compile(r"^Mc([a-z])")
|
||||
PLACEHOLDERS = {"Baby", "Female", "Infant", "Male", "Notnamed", "Unknown", "Unnamed"}
|
||||
|
||||
|
||||
def csv_or_zip_rows(data, member):
|
||||
"""The rows of a CSV, or of every member of a zip named like member, name,sex,count[,year]."""
|
||||
if data.startswith(b"PK"):
|
||||
z = zipfile.ZipFile(io.BytesIO(data))
|
||||
for n in sorted(z.namelist()):
|
||||
if re.fullmatch(member, n):
|
||||
year = re.sub(r"\D", "", n)
|
||||
for r in csv.reader(io.StringIO(z.read(n).decode("utf-8-sig"))):
|
||||
yield r + [year] if year else r
|
||||
return
|
||||
yield from csv.reader(io.StringIO(data.decode("utf-8-sig")))
|
||||
|
||||
|
||||
def given(data, from_year):
|
||||
counts = collections.Counter()
|
||||
for r in csv_or_zip_rows(data, r"yob\d{4}\.txt"):
|
||||
if len(r) >= 4 and r[3].isdigit() and int(r[3]) >= from_year and r[0] not in PLACEHOLDERS:
|
||||
counts[(r[0], r[1].lower())] += int(r[2])
|
||||
return counts
|
||||
|
||||
|
||||
def surname(name):
|
||||
"""A Census uppercase surname as it is written: capitalised, and Mc before a capital."""
|
||||
return MC.sub(lambda m: "Mc" + m.group(1).upper(), name.title())
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("--cache", default=str(CACHE))
|
||||
p.add_argument("--first", type=int, default=2000, help="names per sex")
|
||||
p.add_argument("--from-year", type=int, default=1930)
|
||||
p.add_argument("--last", type=int, default=5000)
|
||||
p.add_argument("--names", default=NAMES)
|
||||
p.add_argument("--out", default=str(OUT))
|
||||
p.add_argument("--surnames", default=SURNAMES)
|
||||
a = p.parse_args()
|
||||
counts = given(source.fetch(a.names, a.cache, "ssa-names.txt"), a.from_year)
|
||||
first = []
|
||||
for sex in ("f", "m"):
|
||||
top = sorted(((n, c) for (n, s), c in counts.items() if s == sex), key=lambda n: (-n[1], n[0]))[:a.first]
|
||||
first += [{"name": n, "sex": sex, "count": c} for n, c in top]
|
||||
rows = csv_or_zip_rows(source.fetch(a.surnames, a.cache, "census-surnames-2010.zip"), r"Names_2010Census\.csv")
|
||||
last = [{"name": surname(r[0]), "count": int(r[2])} for r in rows if len(r) >= 3 and r[2].isdigit() and r[0].isalpha()]
|
||||
last = sorted(last, key=lambda r: (-r["count"], r["name"]))[:a.last]
|
||||
out = Path(a.out)
|
||||
tsv.write(out / "first-name.tsv", ["name", "sex", "count"], sorted(first, key=lambda r: (r["name"], r["sex"])))
|
||||
tsv.write(out / "last-name.tsv", ["name", "count"], sorted(last, key=lambda r: r["name"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,55 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild data/misc/port.tsv from the IANA Service Name and Transport Protocol Port Number Registry.
|
||||
|
||||
data-import/port.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
|
||||
|
||||
A row needs a service name and a numeric TCP port. A port the registry lists more than
|
||||
once keeps the first service it describes, or the first of them where it describes none,
|
||||
so a number selects one row.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import source
|
||||
import tsv
|
||||
|
||||
SOURCE = "https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.csv"
|
||||
OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "port.tsv"
|
||||
CACHE = Path(__file__).resolve().parent / "cache"
|
||||
COLUMNS = ["number", "service"]
|
||||
FLOOR = 4000
|
||||
|
||||
|
||||
def rows(text):
|
||||
best, described = {}, set()
|
||||
for r in csv.DictReader(io.StringIO(text)):
|
||||
service, number = (r["Service Name"] or "").strip(), (r["Port Number"] or "").strip()
|
||||
transport, description = (r["Transport Protocol"] or "").strip(), (r["Description"] or "").strip()
|
||||
if transport != "tcp" or not service or not number.isdigit():
|
||||
continue
|
||||
if not 1 <= int(number) <= 65535:
|
||||
sys.exit(f"{service}: port {number} is outside 1-65535")
|
||||
if number not in best or (description and number not in described):
|
||||
best[number] = {"number": number, "service": service}
|
||||
if description:
|
||||
described.add(number)
|
||||
return sorted(best.values(), key=lambda r: int(r["number"]))
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("--cache", default=str(CACHE))
|
||||
p.add_argument("--source", default=SOURCE)
|
||||
p.add_argument("--out", default=str(OUT))
|
||||
a = p.parse_args()
|
||||
table = rows(source.fetch(a.source, a.cache, "service-names-port-numbers.csv").decode("utf-8"))
|
||||
if len(table) < FLOOR:
|
||||
sys.exit(f"only {len(table)} ports named a TCP service; the registry's columns have moved")
|
||||
tsv.write(a.out, COLUMNS, table)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,67 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild data/misc/protocol.tsv from the IANA Protocol Numbers registry.
|
||||
|
||||
data-import/protocol.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
|
||||
|
||||
A row needs a number and a keyword, which drops the seven numbers assigned to a class of
|
||||
protocols rather than to one; a keyword the registry reserves or marks deprecated goes
|
||||
too. Where the registry names no protocol beside the keyword, the keyword is the name.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import source
|
||||
import tsv
|
||||
|
||||
SOURCE = "https://www.iana.org/assignments/protocol-numbers/protocol-numbers-1.csv"
|
||||
OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "protocol.tsv"
|
||||
CACHE = Path(__file__).resolve().parent / "cache"
|
||||
COLUMNS = ["keyword", "name", "number"]
|
||||
SKIP = ("Reserved", "deprecated")
|
||||
SELECTOR = '[]{}"|' # mirrors inSelector in internal/grammar/path.go
|
||||
|
||||
|
||||
def rows(text):
|
||||
for r in csv.DictReader(io.StringIO(text)):
|
||||
number, keyword = (r["Decimal"] or "").strip(), (r["Keyword"] or "").strip()
|
||||
if not number.isdigit() or not keyword or any(s in keyword for s in SKIP):
|
||||
continue
|
||||
if not 0 <= int(number) <= 255:
|
||||
sys.exit(f"{keyword}: protocol number {number} is outside 0-255")
|
||||
yield {"keyword": keyword, "name": " ".join((r["Protocol"] or "").split()) or keyword, "number": number}
|
||||
|
||||
|
||||
def refuse_an_unselectable_row(table):
|
||||
"""What the loader refuses at New, and a name that names two rows rather than one."""
|
||||
keywords, names = {}, {}
|
||||
for r in table:
|
||||
for column in ("keyword", "name"):
|
||||
if set(r[column]) & set(SELECTOR):
|
||||
sys.exit(f"{r['keyword']}: {column} {r[column]!r} holds one of {SELECTOR}, which a selector cannot spell")
|
||||
if r["keyword"] in keywords:
|
||||
sys.exit(f"{r['keyword']}: repeats the keyword of protocol {keywords[r['keyword']]}")
|
||||
keywords[r["keyword"]] = r["number"]
|
||||
for r in table:
|
||||
if r["name"] != r["keyword"] and r["name"] in keywords:
|
||||
sys.exit(f"{r['keyword']}: name {r['name']!r} is another protocol's keyword")
|
||||
if r["name"] in names:
|
||||
sys.exit(f"{r['keyword']}: name {r['name']!r} repeats {names[r['name']]}'s, so it selects neither")
|
||||
names[r["name"]] = r["keyword"]
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("--cache", default=str(CACHE))
|
||||
p.add_argument("--source", default=SOURCE)
|
||||
p.add_argument("--out", default=str(OUT))
|
||||
a = p.parse_args()
|
||||
table = sorted(rows(source.fetch(a.source, a.cache, "protocol-numbers-1.csv").decode("utf-8")), key=lambda r: int(r["number"]))
|
||||
refuse_an_unselectable_row(table)
|
||||
tsv.write(a.out, COLUMNS, table)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,29 +0,0 @@
|
||||
"""A source fetched once into the cache."""
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def fetch(source, cache, name, magic=b"", data=None, headers=None):
|
||||
"""The bytes of a URL, downloaded into cache/name once, or of a local file."""
|
||||
if not re.match(r"^https?://", source):
|
||||
return Path(source).read_bytes()
|
||||
path = Path(cache) / name
|
||||
if path.exists():
|
||||
return path.read_bytes()
|
||||
for attempt in range(1, 6):
|
||||
req = urllib.request.Request(source, data=data, headers={"User-Agent": "fejkdata-data-import (https://github.com/larvit/fejkdata)", **(headers or {})})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=600) as r:
|
||||
body = r.read()
|
||||
except OSError:
|
||||
body = b""
|
||||
if body and body.startswith(magic) and b"Request Rejected" not in body[:512]:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(body)
|
||||
return body
|
||||
if attempt < 5:
|
||||
time.sleep(10 * attempt)
|
||||
sys.exit(f"{source}: no valid download in 5 attempts")
|
||||
@@ -1,85 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild data/misc/territory.tsv from datasets/country-codes (PDDL).
|
||||
|
||||
data-import/territory.py [--source URL_OR_FILE] [--cache DIR] [--out FILE]
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import source
|
||||
import tsv
|
||||
|
||||
SOURCE = "https://raw.githubusercontent.com/datasets/country-codes/main/data/country-codes.csv"
|
||||
OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "territory.tsv"
|
||||
CACHE = Path(__file__).resolve().parent / "cache"
|
||||
COLUMNS = ["alpha2", "alpha3", "calling-code", "capital", "country", "currency", "flag", "languages", "name", "numeric", "tld"]
|
||||
SOVEREIGN = re.compile(r"^(?:Part of|Territor(?:y|ies) of|Crown dependency of|Commonwealth of|Associated with) ([A-Z]{2})$")
|
||||
STANDALONE = {"In contention", "International"}
|
||||
# Gaps in the source, keyed by alpha2.
|
||||
FIXUPS = {"TR": {"currency": "TRY"}}
|
||||
|
||||
|
||||
def flag(alpha2):
|
||||
return "".join(chr(0x1F1E6 + ord(c) - ord("A")) for c in alpha2)
|
||||
|
||||
|
||||
def calling_code(dial):
|
||||
first = re.sub(r"[^0-9-]", "", dial.split(",")[0])
|
||||
return first if re.match(r"^\d+-\d{3}$", first) else first.split("-")[0]
|
||||
|
||||
|
||||
def languages(field):
|
||||
return ",".join(p for p in field.split(",") if p)
|
||||
|
||||
|
||||
def country(alpha2, independent):
|
||||
"""The sovereign state the register records; a territory it records none for stands alone."""
|
||||
m = SOVEREIGN.match(independent)
|
||||
if m:
|
||||
return m.group(1)
|
||||
if independent == "Yes" or independent in STANDALONE:
|
||||
return alpha2
|
||||
sys.exit(f"{alpha2}: is_independent {independent!r} names no sovereign this script can read")
|
||||
|
||||
|
||||
def rows(text):
|
||||
for r in csv.DictReader(io.StringIO(text)):
|
||||
alpha2 = r["ISO3166-1-Alpha-2"]
|
||||
row = {
|
||||
"alpha2": alpha2,
|
||||
"alpha3": r["ISO3166-1-Alpha-3"],
|
||||
"calling-code": calling_code(r["Dial"]),
|
||||
"capital": r["Capital"],
|
||||
"country": country(alpha2, r["is_independent"]),
|
||||
"currency": r["ISO4217-currency_alphabetic_code"].split(",")[0],
|
||||
"flag": flag(alpha2),
|
||||
"languages": languages(r["Languages"]),
|
||||
"name": r["CLDR display name"] or r["official_name_en"],
|
||||
"numeric": r["ISO3166-1-numeric"].zfill(3),
|
||||
"tld": r["TLD"],
|
||||
}
|
||||
row.update(FIXUPS.get(alpha2, {}))
|
||||
if all(row[c] for c in ("alpha2", "alpha3", "calling-code", "capital", "currency", "name", "tld")):
|
||||
yield row
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("--cache", default=str(CACHE))
|
||||
p.add_argument("--source", default=SOURCE)
|
||||
p.add_argument("--out", default=str(OUT))
|
||||
a = p.parse_args()
|
||||
table = sorted(rows(source.fetch(a.source, a.cache, "country-codes.csv").decode("utf-8")), key=lambda r: r["alpha2"])
|
||||
shipped = {r["alpha2"] for r in table}
|
||||
for r in table:
|
||||
if r["country"] not in shipped:
|
||||
sys.exit(f"{r['alpha2']}: country {r['country']} is no shipped territory; a sovereign the set drops leaves its territories unreadable")
|
||||
tsv.write(a.out, COLUMNS, table)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,138 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild data/misc/timezone.tsv from the IANA tzdb tarball (public domain) and GeoNames (CC BY 4.0).
|
||||
|
||||
data-import/timezone.py [--source URL_OR_FILE] [--cities URL_OR_FILE] [--cache DIR] [--out FILE] [--territories FILE]
|
||||
|
||||
The offset is the zone's standard offset, the first field of its Zone rule's last
|
||||
continuation line, with a Link resolved to its target. The weight is the population
|
||||
GeoNames records in the zone's cities of 15,000 or more, never below that threshold,
|
||||
so a drawn zone is one people live in.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
import sys
|
||||
import tarfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import source
|
||||
import tsv
|
||||
|
||||
SOURCE = "https://data.iana.org/time-zones/tzdata-latest.tar.gz"
|
||||
CITIES = "https://download.geonames.org/export/dump/cities15000.zip"
|
||||
OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "timezone.tsv"
|
||||
TERRITORIES = Path(__file__).resolve().parent.parent / "data" / "misc" / "territory.tsv"
|
||||
CACHE = Path(__file__).resolve().parent / "cache"
|
||||
COLUMNS = ["offset", "territory", "weight", "zone"]
|
||||
REGIONS = ["africa", "antarctica", "asia", "australasia", "backward", "etcetera", "europe", "northamerica", "southamerica"]
|
||||
# GeoNames' own cutoff, so a zone it records no city in weighs less than one would.
|
||||
FLOOR = 15000
|
||||
|
||||
|
||||
def member(tar, name):
|
||||
try:
|
||||
f = tar.extractfile(name)
|
||||
except KeyError:
|
||||
f = None
|
||||
if f is None:
|
||||
sys.exit(f"{name}: the tarball no longer holds it as a file; the tzdb layout has moved")
|
||||
return f.read().decode("utf-8")
|
||||
|
||||
|
||||
def offsets(tar):
|
||||
"""Every zone's standard offset, and every link's target."""
|
||||
std, links = {}, {}
|
||||
for name in REGIONS:
|
||||
zone = None
|
||||
for raw in member(tar, name).splitlines():
|
||||
line = raw.split("#")[0].rstrip()
|
||||
if not line.strip():
|
||||
continue
|
||||
if line.startswith("Zone"):
|
||||
f = line.split()
|
||||
zone, std[f[1]] = f[1], f[2]
|
||||
elif line.startswith("Link"):
|
||||
f = line.split()
|
||||
links[f[2]], zone = f[1], None
|
||||
elif line[0] in " \t" and zone:
|
||||
std[zone] = line.split()[0]
|
||||
else:
|
||||
zone = None
|
||||
return std, links
|
||||
|
||||
|
||||
def resolve(zone, std, links):
|
||||
for _ in range(10):
|
||||
if zone in std:
|
||||
return std[zone]
|
||||
if zone not in links:
|
||||
return None
|
||||
zone = links[zone]
|
||||
return None
|
||||
|
||||
|
||||
def utc_offset(raw):
|
||||
"""±HH:MM from a tzdb STDOFF field, which writes the minutes and seconds only when it has them."""
|
||||
m = re.match(r"^(-)?(\d{1,2})(?::(\d{2})(?::(\d{2}))?)?$", raw)
|
||||
if not m or (m.group(4) or "00") != "00":
|
||||
return None
|
||||
return f"{'-' if m.group(1) else '+'}{int(m.group(2)):02d}:{m.group(3) or '00'}"
|
||||
|
||||
|
||||
def population(body):
|
||||
"""The population GeoNames records per timezone, summed over its cities."""
|
||||
with zipfile.ZipFile(io.BytesIO(body)) as z:
|
||||
text = z.read("cities15000.txt").decode("utf-8")
|
||||
per = {}
|
||||
for city in csv.reader(io.StringIO(text), delimiter="\t", quoting=csv.QUOTE_NONE):
|
||||
if len(city) > 17 and city[17] and city[14].isdigit():
|
||||
per[city[17]] = per.get(city[17], 0) + int(city[14])
|
||||
return per
|
||||
|
||||
|
||||
def rows(tar, territories, pop):
|
||||
std, links = offsets(tar)
|
||||
for line in member(tar, "zone.tab").splitlines():
|
||||
if line.startswith("#") or not line.strip():
|
||||
continue
|
||||
fields = line.split("\t")
|
||||
if len(fields) < 3:
|
||||
sys.exit(f"zone.tab: {line!r} has {len(fields)} fields; a row names a territory, a location and a zone")
|
||||
territory, zone = fields[0], fields[2]
|
||||
if territory not in territories:
|
||||
continue
|
||||
raw = resolve(zone, std, links)
|
||||
if raw is None:
|
||||
sys.exit(f"{zone}: the tarball gives it no Zone rule and no Link to one")
|
||||
offset = utc_offset(raw)
|
||||
if offset is None:
|
||||
sys.exit(f"{zone}: standard offset {raw!r} is not a ±HH:MM the table can spell")
|
||||
yield {"offset": offset, "territory": territory, "weight": max(pop.get(zone, 0), FLOOR), "zone": zone}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("--cache", default=str(CACHE))
|
||||
p.add_argument("--source", default=SOURCE)
|
||||
p.add_argument("--out", default=str(OUT))
|
||||
p.add_argument("--cities", default=CITIES)
|
||||
p.add_argument("--territories", default=str(TERRITORIES))
|
||||
a = p.parse_args()
|
||||
shipped = {r["alpha2"] for r in csv.DictReader(io.StringIO(Path(a.territories).read_text(encoding="utf-8")), delimiter="\t")}
|
||||
pop = population(source.fetch(a.cities, a.cache, "cities15000.zip", magic=b"PK"))
|
||||
body = source.fetch(a.source, a.cache, "tzdata-latest.tar.gz", magic=b"\x1f\x8b")
|
||||
with tarfile.open(fileobj=io.BytesIO(body)) as tar:
|
||||
table = sorted(rows(tar, shipped, pop), key=lambda r: r["zone"])
|
||||
print(f"tzdb {tar.extractfile('version').read().decode('utf-8').strip()}", file=sys.stderr)
|
||||
linked = {r["territory"] for r in table}
|
||||
if missing := shipped - linked:
|
||||
sys.exit(f"no zone for {sorted(missing)}; a parent row without a child is a load error")
|
||||
if (floored := sum(1 for r in table if r["weight"] == FLOOR)) > len(table) // 2:
|
||||
sys.exit(f"{floored} of {len(table)} zones fell to the floor; GeoNames' population or timezone column has moved")
|
||||
tsv.write(a.out, COLUMNS, table)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,68 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild data/misc/tld.tsv from the IANA Root Zone Database.
|
||||
|
||||
data-import/tld.py [--source URL_OR_FILE] [--zone URL_OR_FILE] [--cache DIR] [--out FILE]
|
||||
|
||||
A TLD ships when the database names a manager for it; one it marks "Not assigned" is a
|
||||
delegation the root zone no longer holds, and the zone file is fetched to prove the two
|
||||
agree. The key is the A-label the root zone holds; `unicode` is the form the database
|
||||
displays, less the bidi marks it wraps a right-to-left label in, and proved to punycode
|
||||
back to the key.
|
||||
"""
|
||||
import argparse
|
||||
import html
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import source
|
||||
import tsv
|
||||
|
||||
SOURCE = "https://www.iana.org/domains/root/db"
|
||||
ZONE = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
|
||||
OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "tld.tsv"
|
||||
CACHE = Path(__file__).resolve().parent / "cache"
|
||||
COLUMNS = ["tld", "type", "unicode"]
|
||||
TYPES = ("country-code", "generic", "generic-restricted", "infrastructure", "sponsored")
|
||||
BIDI = {0x200E: None, 0x200F: None}
|
||||
ROW = re.compile(r'<a href="/domains/root/db/([^"]+)\.html">([^<]+)</a>.*?</td>\s*<td>([^<]*)</td>\s*<td>(.*?)</td>', re.S)
|
||||
|
||||
|
||||
def rows(page):
|
||||
for a_label, shown, kind, manager in ROW.findall(page):
|
||||
if "Not assigned" in manager:
|
||||
continue
|
||||
kind = kind.strip()
|
||||
if kind not in TYPES:
|
||||
sys.exit(f"{a_label}: the database types it {kind!r}, which is none of {TYPES}")
|
||||
yield {"tld": "." + a_label, "type": kind, "unicode": html.unescape(shown).strip().translate(BIDI)}
|
||||
|
||||
|
||||
def prove_each_unicode_form_is_its_key(table):
|
||||
"""The displayed form is the key's own spelling, which punycode encodes back to."""
|
||||
for r in table:
|
||||
label = r["unicode"][1:]
|
||||
encoded = label if label.isascii() else "xn--" + label.encode("punycode").decode("ascii")
|
||||
if "." + encoded != r["tld"]:
|
||||
sys.exit(f"{r['tld']}: the database displays {r['unicode']!r}, which is no spelling of it")
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("--cache", default=str(CACHE))
|
||||
p.add_argument("--source", default=SOURCE)
|
||||
p.add_argument("--zone", default=ZONE)
|
||||
p.add_argument("--out", default=str(OUT))
|
||||
a = p.parse_args()
|
||||
table = sorted(rows(source.fetch(a.source, a.cache, "root-db.html").decode("utf-8")), key=lambda r: r["tld"])
|
||||
prove_each_unicode_form_is_its_key(table)
|
||||
listing = source.fetch(a.zone, a.cache, "tlds-alpha-by-domain.txt").decode("utf-8")
|
||||
delegated = {"." + line.strip().lower() for line in listing.splitlines() if line.strip() and not line.startswith("#")}
|
||||
shipped = {r["tld"] for r in table}
|
||||
if len(table) != len(delegated) or shipped != delegated:
|
||||
sys.exit(f"the database assigns {len(table)} rows where the root zone holds {len(delegated)} TLDs: {sorted(shipped ^ delegated)[:10]}")
|
||||
tsv.write(a.out, COLUMNS, table)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,18 +0,0 @@
|
||||
"""A table written as the loader admits it."""
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def write(path, columns, rows):
|
||||
"""Write the rows as a TSV of a header and at least one row, each cell stripped, non-empty and free of tabs, newlines and braces."""
|
||||
lines = ["\t".join(columns)]
|
||||
for row in rows:
|
||||
cells = [str(row[c]).strip() for c in columns]
|
||||
if not all(cells) or any(re.search(r"[\t\n{}]", c) for c in cells):
|
||||
raise ValueError(f"{path}: a cell is empty or holds a tab, newline or brace: {row}")
|
||||
lines.append("\t".join(cells))
|
||||
if len(lines) < 2:
|
||||
raise ValueError(f"{path}: has no rows below its header; a table holds at least one row")
|
||||
Path(path).write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
print(f"{path}: {len(lines) - 1} rows", file=sys.stderr)
|
||||
@@ -1,64 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild data/misc/useragent.tsv from microlinkhq/top-user-agents (MIT).
|
||||
|
||||
data-import/useragent.py [--source URL] [--cache DIR] [--out FILE]
|
||||
|
||||
The desktop and mobile lists partition the whole, so `device` is read rather than
|
||||
parsed. A row ships when the string names a browser and an operating system both.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import source
|
||||
import tsv
|
||||
|
||||
SOURCE = "https://raw.githubusercontent.com/microlinkhq/top-user-agents/master/src"
|
||||
OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "useragent.tsv"
|
||||
CACHE = Path(__file__).resolve().parent / "cache"
|
||||
COLUMNS = ["browser", "device", "os", "ua"]
|
||||
# First match wins: a Chromium fork this list does not name ships as Chrome.
|
||||
BROWSERS = [("Edge", r"Edg(A|iOS)?/"), ("Opera", r"OPR/"), ("Samsung Internet", r"SamsungBrowser/"),
|
||||
("Chrome", r"(Chrome|CriOS)/"), ("Firefox", r"(Firefox|FxiOS)/"), ("Safari", r"Version/[\d.]+ .*Safari")]
|
||||
SYSTEMS = [("iOS", r"iPhone|iPad|CPU OS "), ("Android", r"Android"), ("ChromeOS", r"CrOS"),
|
||||
("Windows", r"Windows NT"), ("macOS", r"Macintosh|Mac OS X"), ("Linux", r"X11.*Linux|Ubuntu")]
|
||||
|
||||
|
||||
def named(table, ua):
|
||||
for name, pattern in table:
|
||||
if re.search(pattern, ua):
|
||||
return name
|
||||
return ""
|
||||
|
||||
|
||||
def rows(desktop, mobile):
|
||||
seen, kept = set(), []
|
||||
for device, uas in (("desktop", desktop), ("mobile", mobile)):
|
||||
for ua in uas:
|
||||
browser, os = named(BROWSERS, ua), named(SYSTEMS, ua)
|
||||
if not browser or not os:
|
||||
print(f"dropped, naming no {'browser' if not browser else 'operating system'}: {ua}", file=sys.stderr)
|
||||
continue
|
||||
if ua in seen:
|
||||
sys.exit(f"{ua}: listed twice, which would draw it twice as often")
|
||||
seen.add(ua)
|
||||
kept.append({"browser": browser, "device": device, "os": os, "ua": ua})
|
||||
if len(kept) < 0.8 * (len(desktop) + len(mobile)):
|
||||
sys.exit(f"only {len(kept)} of {len(desktop) + len(mobile)} strings named both; the tokens have moved")
|
||||
return kept
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("--cache", default=str(CACHE))
|
||||
p.add_argument("--source", default=SOURCE)
|
||||
p.add_argument("--out", default=str(OUT))
|
||||
a = p.parse_args()
|
||||
lists = [json.loads(source.fetch(f"{a.source}/{n}.json", a.cache, f"top-user-agents-{n}.json")) for n in ("desktop", "mobile")]
|
||||
tsv.write(a.out, COLUMNS, sorted(rows(*lists), key=lambda r: (r["browser"], r["os"], r["ua"])))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,21 +0,0 @@
|
||||
"""The rows of an xlsx sheet."""
|
||||
import io
|
||||
import xml.etree.ElementTree as ET
|
||||
import zipfile
|
||||
|
||||
NS = {"m": "http://schemas.openxmlformats.org/spreadsheetml/2006/main", "r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships"}
|
||||
|
||||
|
||||
def rows(data, sheet=None):
|
||||
"""The rows of an xlsx sheet named sheet, the first sheet by default, each a list of cell texts."""
|
||||
z = zipfile.ZipFile(io.BytesIO(data))
|
||||
strings = ["".join(t.text or "" for t in si.iter("{%s}t" % NS["m"])) for si in ET.fromstring(z.read("xl/sharedStrings.xml")).findall("m:si", NS)]
|
||||
rels = {r.get("Id"): r.get("Target") for r in ET.fromstring(z.read("xl/_rels/workbook.xml.rels"))}
|
||||
sheets = {s.get("name"): rels[s.get("{%s}id" % NS["r"])] for s in ET.fromstring(z.read("xl/workbook.xml")).iter("{%s}sheet" % NS["m"])}
|
||||
target = sheets[sheet] if sheet else next(iter(sheets.values()))
|
||||
for row in ET.fromstring(z.read("xl/" + target)).findall(".//m:row", NS):
|
||||
cells = []
|
||||
for c in row.findall("m:c", NS):
|
||||
v = c.find("m:v", NS)
|
||||
cells.append("" if v is None else strings[int(v.text)] if c.get("t") == "s" else v.text)
|
||||
yield cells
|
||||
@@ -1,175 +1,156 @@
|
||||
package fejkdata
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/larvit/fejkdata/internal/datafiles"
|
||||
)
|
||||
|
||||
// loadSources merges sources in order, the last winning a clash. A source whose manifest
|
||||
// carries an index leaves its categories unloaded; every other source loads here, with
|
||||
// the unloaded categories it needs.
|
||||
// docs/decisions.md#a-source-whose-manifest-carries-an-index-loads-each-category-on-first-reach-any-other-loads-in-new-with-what-it-reads
|
||||
func loadSources(sources []datafiles.Source) (folder, error) {
|
||||
root := folder{children: map[string]node{}}
|
||||
for _, src := range sources {
|
||||
m, err := src.Manifest()
|
||||
if err != nil {
|
||||
return folder{}, err
|
||||
}
|
||||
if err := addSource(&root, src, m); err != nil {
|
||||
return folder{}, err
|
||||
}
|
||||
}
|
||||
return root, loadReached(&root, categorySites(&root), nil)
|
||||
// dataSource is one tree to load: an fs.FS and the directory in it to start from.
|
||||
// label prefixes file names in errors; path, when set, is a directory on disk that
|
||||
// must exist.
|
||||
type dataSource struct {
|
||||
fsys fs.FS
|
||||
label string
|
||||
path string
|
||||
root string
|
||||
}
|
||||
|
||||
func addSource(root *folder, src datafiles.Source, m datafiles.Manifest) error {
|
||||
src = src.Mounted(m.Namespace)
|
||||
if m.Index != nil {
|
||||
placeIndex(root, src, m.Index)
|
||||
return nil
|
||||
func (s dataSource) name(p string) string {
|
||||
if s.label == "" {
|
||||
return p
|
||||
}
|
||||
g := &folder{children: map[string]node{}}
|
||||
if err := src.Walk(compileInto(func(dir []string) *folder { return madeFolder(g, dir) })); err != nil {
|
||||
return path.Join(s.label, p)
|
||||
}
|
||||
|
||||
// loadData loads every source into one namespace tree and returns its root
|
||||
// children. A directory becomes a group; each *.json file in it compiles to a node
|
||||
// keyed by its base name (address.json -> "address"); each subdirectory becomes a
|
||||
// nested group, so folders turn into dot-path segments. Sources merge left to right:
|
||||
// matching groups merge by their children, and any other clash is won by the last
|
||||
// source loaded. Once merged, linkRefs binds every reference against the
|
||||
// final tree.
|
||||
func loadData(sources []dataSource) (map[string]node, error) {
|
||||
root := map[string]node{}
|
||||
for _, src := range sources {
|
||||
if src.path != "" {
|
||||
info, err := os.Stat(src.path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: no such directory", src.path)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, fmt.Errorf("%s is not a directory", src.path)
|
||||
}
|
||||
}
|
||||
dir := src.root
|
||||
if dir == "" {
|
||||
dir = "."
|
||||
}
|
||||
g, err := loadDir(src, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mergeChildren(root, g.children)
|
||||
}
|
||||
if len(root) == 0 {
|
||||
return nil, fmt.Errorf("no .json data found")
|
||||
}
|
||||
if err := linkRefs(root); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := checkNoCycles(root); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := checkRepeatReach(root); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := checkBoundLevelsHeld(root); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return root, nil
|
||||
}
|
||||
|
||||
// loadDir compiles one directory into a group. fs.ReadDir yields entries sorted
|
||||
// by name, so the tree is built deterministically. Empty subdirectories (no JSON
|
||||
// anywhere under them) are skipped rather than added as empty namespaces.
|
||||
func loadDir(src dataSource, dir string) (*group, error) {
|
||||
entries, err := fs.ReadDir(src.fsys, dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", src.name(dir), err)
|
||||
}
|
||||
g := &group{children: map[string]node{}}
|
||||
for _, e := range entries {
|
||||
if strings.HasPrefix(e.Name(), ".") { // hidden: a checkout or an editor's file, never data
|
||||
continue
|
||||
}
|
||||
full := path.Join(dir, e.Name())
|
||||
load := loadFile
|
||||
if e.IsDir() {
|
||||
load = loadFolder
|
||||
}
|
||||
if err := load(src, g, full, e.Name()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return g, nil
|
||||
}
|
||||
|
||||
// loadFolder adds a subdirectory as a nested group, unless nothing under it is data.
|
||||
func loadFolder(src dataSource, g *group, full, name string) error {
|
||||
child, err := loadDir(src, full)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mergeFolder(root, g)
|
||||
if len(child.children) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := checkName(name); err != nil {
|
||||
return fmt.Errorf("%s: folder %w", src.name(full), err)
|
||||
}
|
||||
g.children[name] = child
|
||||
return nil
|
||||
}
|
||||
|
||||
func compileInto(place func(dir []string) *folder) func(datafiles.Category) error {
|
||||
return func(c datafiles.Category) error {
|
||||
n, err := compileCategory(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
place(c.Folders).put(c.Name, n)
|
||||
// loadFile compiles a *.json file into a category named after it; any other file
|
||||
// is skipped.
|
||||
func loadFile(src dataSource, g *group, full, file string) error {
|
||||
if !strings.HasSuffix(file, ".json") {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// madeFolder is the folder that dir names under root, made where it is missing or a category.
|
||||
func madeFolder(root *folder, dir []string) *folder {
|
||||
g := root
|
||||
for _, seg := range dir {
|
||||
sub, isFolder := g.children[seg].(*folder)
|
||||
if !isFolder {
|
||||
sub = &folder{children: map[string]node{}}
|
||||
g.put(seg, sub)
|
||||
}
|
||||
g = sub
|
||||
name := strings.TrimSuffix(file, ".json")
|
||||
if err := checkName(name); err != nil {
|
||||
return fmt.Errorf("%s: category %w", src.name(full), err)
|
||||
}
|
||||
return g
|
||||
b, err := fs.ReadFile(src.fsys, full)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", src.name(full), err)
|
||||
}
|
||||
var raw any
|
||||
if err := json.Unmarshal(b, &raw); err != nil {
|
||||
return fmt.Errorf("%s: %w", src.name(full), err)
|
||||
}
|
||||
n, err := compile(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", src.name(full), err)
|
||||
}
|
||||
g.children[name] = n
|
||||
return nil
|
||||
}
|
||||
|
||||
func mergeFolder(dst, src *folder) {
|
||||
for k, v := range src.children {
|
||||
if dg, ok := dst.children[k].(*folder); ok {
|
||||
if sg, ok := v.(*folder); ok {
|
||||
mergeFolder(dg, sg)
|
||||
// mergeChildren overlays src onto dst. Two groups under the same key merge
|
||||
// recursively (so locales/categories from several paths combine); every other
|
||||
// key is replaced, making the last-loaded directory win on a conflict.
|
||||
func mergeChildren(dst, src map[string]node) {
|
||||
for k, v := range src {
|
||||
if dg, ok := dst[k].(*group); ok {
|
||||
if sg, ok := v.(*group); ok {
|
||||
mergeChildren(dg.children, sg.children)
|
||||
continue
|
||||
}
|
||||
}
|
||||
dst.put(k, v)
|
||||
dst[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
// nodeSet is the set of nodes one validation pass covers: one load's categories, or a
|
||||
// single inline node.
|
||||
type nodeSet func(fn func(label string, n node) error) error
|
||||
|
||||
// categorySite is a loaded category, where it sits and its own dot path.
|
||||
type categorySite struct {
|
||||
categoryAt
|
||||
n node
|
||||
path string
|
||||
}
|
||||
|
||||
// categorySites lists every category under root, folders and names in sorted order.
|
||||
func categorySites(root *folder) []categorySite {
|
||||
var out []categorySite
|
||||
var walk func(dir []string, g *folder)
|
||||
walk = func(dir []string, g *folder) {
|
||||
for _, name := range sortedNames(g.children) {
|
||||
if sub, isFolder := g.children[name].(*folder); isFolder {
|
||||
walk(append(dir[:len(dir):len(dir)], name), sub)
|
||||
continue
|
||||
}
|
||||
out = append(out, siteIn(dir, g, name))
|
||||
}
|
||||
}
|
||||
walk(nil, root)
|
||||
return out
|
||||
}
|
||||
|
||||
func siteIn(dir []string, in *folder, name string) categorySite {
|
||||
return categorySite{categoryAt: categoryAt{dir: dir, in: in, name: name}, n: in.children[name], path: categoryPath(dir, name)}
|
||||
}
|
||||
|
||||
func categoryPath(dir []string, name string) string { return join(strings.Join(dir, "."), name) }
|
||||
|
||||
func siteNodes(sites []categorySite) nodeSet {
|
||||
return func(fn func(label string, n node) error) error {
|
||||
for _, s := range sites {
|
||||
if err := eachNode(s.n, s.path, fn); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func inlineNodes(n node, label string) nodeSet {
|
||||
return func(fn func(label string, m node) error) error { return eachNode(n, label, fn) }
|
||||
}
|
||||
|
||||
// pipeline is the one load pipeline, run over one nodeSet; its fields are where
|
||||
// entry points differ.
|
||||
type pipeline struct {
|
||||
nodes nodeSet
|
||||
resolve func() error
|
||||
// Set by a caller that proves the columns against their Go types itself.
|
||||
typedByGo bool
|
||||
}
|
||||
|
||||
// categoryPipeline is the pipeline over the categories of one load, their references
|
||||
// resolving against root.
|
||||
func categoryPipeline(sites []categorySite, root map[string]node) pipeline {
|
||||
return pipeline{
|
||||
nodes: siteNodes(sites),
|
||||
resolve: func() error {
|
||||
if err := linkTables(sites); err != nil {
|
||||
return err
|
||||
}
|
||||
return resolveCategoryTemplates(sites, root)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (p pipeline) run() error {
|
||||
if err := p.resolve(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkNoCycles(p.nodes); err != nil {
|
||||
return err
|
||||
}
|
||||
settleRecords(p.nodes)
|
||||
if !p.typedByGo {
|
||||
if err := checkColumns(p.nodes); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return checkNodeFences(p.nodes)
|
||||
}
|
||||
|
||||
// checkNodeFences runs the per-node fences over s, each over every node before the next,
|
||||
// so which of several broken nodes is reported does not depend on the walk. Its walks
|
||||
// recurse unguarded, so pipeline.run refuses cycles first.
|
||||
func checkNodeFences(s nodeSet) error {
|
||||
mem := renderCounts{}
|
||||
if err := s(func(label string, n node) error { return repeatCheck(label, n, mem) }); err != nil {
|
||||
return err
|
||||
}
|
||||
return s((&valueProof{}).checkDatatype)
|
||||
}
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
{
|
||||
"index": {
|
||||
"en_US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
|
||||
"en_US.color": {"paths":[""]},
|
||||
"en_US.company": {"paths":["","base","suffix"]},
|
||||
"en_US.date": {"paths":[""]},
|
||||
"en_US.email": {"paths":["","domain","local","local.n"]},
|
||||
"en_US.first-name": {"parent":"sex","paths":["","count","name","sex"]},
|
||||
"en_US.ip": {"paths":[""]},
|
||||
"en_US.itin": {"paths":["","area","group","serial"]},
|
||||
"en_US.last-name": {"paths":["","count","name"]},
|
||||
"en_US.person": {"paths":["","first","last","prefix","sex"]},
|
||||
"en_US.phone": {"paths":["","area","exchange","line"]},
|
||||
"en_US.phone-exchange": {"paths":[""]},
|
||||
"en_US.price": {"paths":["","amt","cents"]},
|
||||
"en_US.sentence": {"paths":["","adj","noun","prep","verb"]},
|
||||
"en_US.sex": {"paths":["","code","first-name","first-name.count","first-name.name","first-name.sex","name","title","title.name","title.sex","title.share"]},
|
||||
"en_US.ssn": {"paths":["","area","group","serial"]},
|
||||
"en_US.time": {"paths":[""]},
|
||||
"en_US.title": {"parent":"sex","paths":["","name","sex","share"]},
|
||||
"en_US.url": {"paths":["","host","path"]},
|
||||
"en_US.username": {"paths":["","n"]},
|
||||
"en_US.version": {"paths":["","n","pre","suffix"]},
|
||||
"en_US.word": {"paths":[""]}
|
||||
},
|
||||
"namespace": "en_US"
|
||||
}
|
||||
+15
-6
@@ -1,8 +1,17 @@
|
||||
{
|
||||
"format": "{/geo.US.address as a}{a}",
|
||||
"street": "{a.street}",
|
||||
"street-number": "{a.street-number}",
|
||||
"locality": "{a.locality}",
|
||||
"region": "{a.region}",
|
||||
"postal-code": "{a.postal-code}"
|
||||
"format": "{street-number} {street}\n{locality}, {region} {postal-code}",
|
||||
"street": {
|
||||
"format": "{name} {suffix}",
|
||||
"name": ["Adams", "Ashby", "Aspen", "Bay", "Birch", "Bridge", "Cedar", "Chestnut", "Church", "Clark", "Cypress", "Dogwood", "Elm", "Forest", "Franklin", "Garden", "Grove", "Hawthorn", "Hickory", "Highland", "Jackson", "Jefferson", "Juniper", "Lake", "Laurel", "Liberty", "Lincoln", "Madison", "Magnolia", "Maple", "Market", "Meadow", "Mill", "Oak", "Park", "Pine", "Poplar", "Prospect", "Ridge", "River", "Spruce", "Sunset", "Sycamore", "Union", "Walnut", "Washington", "Willow", "Wilson"],
|
||||
"suffix": ["Avenue", "Boulevard", "Circle", "Court", "Drive", "Lane", "Place", "Road", "Street", "Terrace", "Trail", "Way"]
|
||||
},
|
||||
"street-number": [
|
||||
"{int(10,99)}",
|
||||
{ "format": "{int(100,999)}", "weight": 2 },
|
||||
{ "format": "{int(1000,9999)}", "weight": 0.5 },
|
||||
{ "format": "{int(10,99)}{int(100,999)}", "weight": 0.3 }
|
||||
],
|
||||
"locality": ["Albany", "Atlanta", "Austin", "Baltimore", "Boston", "Charlotte", "Chicago", "Cincinnati", "Cleveland", "Columbus", "Dallas", "Denver", "Detroit", "El Paso", "Fort Worth", "Fresno", "Houston", "Indianapolis", "Jacksonville", "Kansas City", "Las Vegas", "Long Beach", "Los Angeles", "Memphis", "Mesa", "Miami", "Milwaukee", "Minneapolis", "Nashville", "New Orleans", "Oakland", "Oklahoma City", "Omaha", "Orlando", "Philadelphia", "Phoenix", "Pittsburgh", "Portland", "Raleigh", "Sacramento", "San Antonio", "San Diego", "San Jose", "Seattle", "St. Louis", "Tampa", "Tucson", "Tulsa"],
|
||||
"region": ["AL", "AZ", "CA", "CO", "CT", "FL", "GA", "IL", "IN", "KY", "LA", "MA", "MD", "MI", "MN", "MO", "NC", "NJ", "NV", "NY", "OH", "OK", "OR", "PA", "TN", "TX", "VA", "WA", "WI"],
|
||||
"postal-code": ["{int(10000,99999)}", { "format": "{int(10000,99999)}-{digits(4)}", "weight": 0.3 }]
|
||||
}
|
||||
|
||||
+13
-1
@@ -1 +1,13 @@
|
||||
"{date(1970-01-01,2029-12-31,'01/02/2006')}"
|
||||
{
|
||||
"format": "{month}/{day}/{year}",
|
||||
"month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"],
|
||||
"day": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28"],
|
||||
"year": [
|
||||
"197{digits(1)}",
|
||||
"198{digits(1)}",
|
||||
{ "format": "199{digits(1)}", "weight": 2 },
|
||||
{ "format": "200{digits(1)}", "weight": 3 },
|
||||
{ "format": "201{digits(1)}", "weight": 3 },
|
||||
{ "format": "202{digits(1)}", "weight": 2 }
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
{ "format": "{name}", "rows": "first-name.tsv", "name": "name", "parent": "sex", "weight": "count" }
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,19 +0,0 @@
|
||||
// Package enus is fejkdata's en_US locale module, whose categories are en_US.*; its address reads
|
||||
// geo.US, which package geous holds.
|
||||
package enus
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"io/fs"
|
||||
|
||||
geous "github.com/larvit/fejkdata/data/geo/US"
|
||||
)
|
||||
|
||||
// FS holds the locale alone, its categories and its manifest; pass Modules unless your own
|
||||
// data stands in for geo.US.
|
||||
//
|
||||
//go:embed *.json *.tsv .fejkdata.json
|
||||
var FS embed.FS
|
||||
|
||||
// Modules returns FS and every module it reads, to pass to fejkdata.WithDataFS.
|
||||
func Modules() []fs.FS { return append([]fs.FS{FS}, geous.Modules()...) }
|
||||
@@ -1,22 +0,0 @@
|
||||
package enus_test
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"testing"
|
||||
|
||||
"github.com/larvit/fejkdata"
|
||||
enus "github.com/larvit/fejkdata/data/en_US"
|
||||
)
|
||||
|
||||
func TestFSLoadsByItsManifest(t *testing.T) {
|
||||
if _, err := fs.Stat(enus.FS, ".fejkdata.json"); err != nil {
|
||||
t.Fatalf("FS holds no manifest: %v", err)
|
||||
}
|
||||
f, err := fejkdata.New(fejkdata.WithDataFS(enus.Modules()...), fejkdata.WithSeed(1))
|
||||
if err != nil {
|
||||
t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
|
||||
}
|
||||
if len(f.List()) == 0 {
|
||||
t.Error("List() is empty, want the module's categories")
|
||||
}
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
{
|
||||
"format": "{area}-{group}-{serial}",
|
||||
"area": "9{digits(2)}",
|
||||
"group": [
|
||||
{ "format": "{int(50,65)}", "weight": 16 },
|
||||
{ "format": "{int(70,88)}", "weight": 19 },
|
||||
{ "format": "{int(90,92)}", "weight": 3 },
|
||||
{ "format": "{int(94,99)}", "weight": 6 }
|
||||
],
|
||||
"serial": [
|
||||
{ "format": "000{int(1,9)}", "weight": 9 },
|
||||
{ "format": "00{int(10,99)}", "weight": 90 },
|
||||
{ "format": "0{int(100,999)}", "weight": 900 },
|
||||
{ "format": "{int(1000,9999)}", "weight": 9000 }
|
||||
]
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
{ "format": "{name}", "rows": "last-name.tsv", "key": "name", "weight": "count" }
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"format": "{prefix}{first} {last}",
|
||||
"first": "{.first-name as n}{n.name}",
|
||||
"last": "{.last-name.name}",
|
||||
"prefix": [null, { "format": "{n..sex.title.name} ", "weight": 0.1 }],
|
||||
"sex": "{n..sex.name}"
|
||||
"format": "{prefix}{femalefirst|malefirst} {last}",
|
||||
"femalefirst": ["Abigail", "Addison", "Amelia", "Aria", "Aubrey", "Audrey", "Aurora", "Ava", "Bella", "Brooklyn", "Camila", "Caroline", "Charlotte", "Chloe", "Claire", "Eleanor", "Elizabeth", "Ella", "Ellie", "Emily", "Emma", "Evelyn", "Gianna", "Grace", "Hannah", "Harper", "Hazel", "Isabella", "Layla", "Leah", "Lillian", "Lily", "Lucy", "Luna", "Madison", "Maya", "Mia", "Mila", "Naomi", "Natalie", "Nora", "Olivia", "Paisley", "Penelope", "Riley", "Savannah", "Scarlett", "Sofia", "Sophia", "Stella", "Victoria", "Violet", "Zoe"],
|
||||
"malefirst": ["Aiden", "Alexander", "Andrew", "Anthony", "Asher", "Benjamin", "Caleb", "Carter", "Charles", "Christopher", "Daniel", "David", "Dylan", "Elijah", "Ethan", "Ezra", "Gabriel", "Grayson", "Henry", "Isaac", "Jack", "Jackson", "Jacob", "James", "Jayden", "John", "Joseph", "Joshua", "Julian", "Levi", "Liam", "Lincoln", "Logan", "Lucas", "Luke", "Mason", "Mateo", "Matthew", "Michael", "Nathan", "Noah", "Oliver", "Owen", "Samuel", "Sebastian", "Theodore", "Thomas", "William", "Wyatt"],
|
||||
"last": ["Adams", "Allen", "Anderson", "Bailey", "Baker", "Bell", "Bennett", "Brooks", "Brown", "Campbell", "Carter", "Clark", "Collins", "Cook", "Cooper", "Cox", "Davis", "Edwards", "Evans", "Flores", "Foster", "Garcia", "Gonzalez", "Gray", "Green", "Hall", "Harris", "Hayes", "Hernandez", "Hill", "Howard", "Hughes", "Jackson", "James", "Jenkins", "Johnson", "Jones", "Kelly", "King", "Lee", "Lewis", "Long", "Lopez", "Martin", "Martinez", "Miller", "Mitchell", "Moore", "Morgan", "Morris", "Murphy", "Nelson", "Parker", "Perez", "Perry", "Peterson", "Phillips", "Powell", "Price", "Ramirez", "Reed", "Richardson", "Rivera", "Roberts", "Robinson", "Rodriguez", "Rogers", "Ross", "Russell", "Sanchez", "Sanders", "Scott", "Smith", "Stewart", "Sullivan", "Taylor", "Thomas", "Thompson", "Torres", "Turner", "Walker", "Ward", "Watson", "White", "Williams", "Wilson", "Wood", "Wright", "Young"],
|
||||
"prefix": ["", { "format": "{title} ", "title": ["Dr", "Miss", "Mr", "Mrs", "Ms", "Mx", "Prof"], "weight": 0.1 }]
|
||||
}
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
[
|
||||
{ "format": "{int(2,9)}0{digits(1)}", "weight": 80 },
|
||||
{ "format": "{int(2,9)}1{int(2,9)}", "weight": 64 },
|
||||
{ "format": "{int(2,9)}10", "weight": 8 },
|
||||
{ "format": "{int(2,4)}{int(2,9)}{digits(1)}", "weight": 240 },
|
||||
{ "format": "{int(6,8)}{int(2,9)}{digits(1)}", "weight": 240 },
|
||||
{ "format": "5{int(2,4)}{digits(1)}", "weight": 30 },
|
||||
{ "format": "5{int(6,9)}{digits(1)}", "weight": 40 },
|
||||
{ "format": "55{int(0,4)}", "weight": 5 },
|
||||
{ "format": "55{int(6,9)}", "weight": 4 },
|
||||
{ "format": "9{int(2,4)}{digits(1)}", "weight": 30 },
|
||||
{ "format": "9{int(8,9)}{digits(1)}", "weight": 20 },
|
||||
{ "format": "95{int(1,7)}", "weight": 7 },
|
||||
{ "format": "96{digits(1)}", "weight": 10 },
|
||||
{ "format": "97{int(0,5)}", "weight": 6 },
|
||||
{ "format": "97{int(7,9)}", "weight": 3 }
|
||||
]
|
||||
@@ -1,15 +1,15 @@
|
||||
[
|
||||
{
|
||||
"format": "({area}) {exchange}-{line}",
|
||||
"format": "({area}) {exch}-{line}",
|
||||
"weight": 2,
|
||||
"area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
|
||||
"exchange": "{.phone-exchange}",
|
||||
"exch": "{int(100,999)}",
|
||||
"line": "{digits(4)}"
|
||||
},
|
||||
{
|
||||
"format": "{area}-{exchange}-{line}",
|
||||
"format": "{area}-{exch}-{line}",
|
||||
"area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
|
||||
"exchange": "{.phone-exchange}",
|
||||
"exch": "{int(100,999)}",
|
||||
"line": "{digits(4)}"
|
||||
}
|
||||
]
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
{ "format": "{name}", "rows": "sex.tsv", "key": "code", "name": "name" }
|
||||
@@ -1,3 +0,0 @@
|
||||
code name
|
||||
f female
|
||||
m male
|
||||
|
+1
-19
@@ -1,19 +1 @@
|
||||
{
|
||||
"format": "{area}-{group}-{serial}",
|
||||
"area": [
|
||||
{ "format": "00{int(1,9)}", "weight": 9 },
|
||||
{ "format": "0{int(10,99)}", "weight": 90 },
|
||||
{ "format": "{int(100,665)}", "weight": 566 },
|
||||
{ "format": "{int(667,899)}", "weight": 233 }
|
||||
],
|
||||
"group": [
|
||||
{ "format": "0{int(1,9)}", "weight": 9 },
|
||||
{ "format": "{int(10,99)}", "weight": 90 }
|
||||
],
|
||||
"serial": [
|
||||
{ "format": "000{int(1,9)}", "weight": 9 },
|
||||
{ "format": "00{int(10,99)}", "weight": 90 },
|
||||
{ "format": "0{int(100,999)}", "weight": 900 },
|
||||
{ "format": "{int(1000,9999)}", "weight": 9000 }
|
||||
]
|
||||
}
|
||||
"{int(100,999)}-{digits(2)}-{digits(4)}"
|
||||
|
||||
@@ -1 +1,6 @@
|
||||
"{time('3:04 PM')}"
|
||||
{
|
||||
"format": "{hour}:{minute} {ampm}",
|
||||
"hour": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
|
||||
"minute": { "format": "{t}{digits(1)}", "t": ["0", "1", "2", "3", "4", "5"] },
|
||||
"ampm": ["AM", "PM"]
|
||||
}
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
{ "format": "{name}", "rows": "title.tsv", "name": "name", "parent": "sex", "weight": "share" }
|
||||
@@ -1,11 +0,0 @@
|
||||
name sex share
|
||||
Dr f 6
|
||||
Dr m 6
|
||||
Miss f 4
|
||||
Mr m 60
|
||||
Mrs f 22
|
||||
Ms f 30
|
||||
Mx f 1
|
||||
Mx m 1
|
||||
Prof f 3
|
||||
Prof m 3
|
||||
|
@@ -1,11 +0,0 @@
|
||||
{
|
||||
"index": {
|
||||
"geo.SE.address": {"paths":["","locality","postal-code","street","street-number"]},
|
||||
"geo.SE.locality": {"parent":"municipality","paths":["","lat","lon","municipality","name","population","postal-code","postal-code.code","postal-code.locality","street","street.locality","street.name","street.segments"]},
|
||||
"geo.SE.municipality": {"parent":"region","paths":["","code","locality","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.locality","locality.street.name","locality.street.segments","name","population","region"]},
|
||||
"geo.SE.postal-code": {"parent":"locality","paths":["","code","locality"]},
|
||||
"geo.SE.region": {"paths":["","code","municipality","municipality.code","municipality.locality","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.locality","municipality.locality.street.name","municipality.locality.street.segments","municipality.name","municipality.population","municipality.region","name","population","timezone"]},
|
||||
"geo.SE.street": {"parent":"locality","paths":["","locality","name","segments"]}
|
||||
},
|
||||
"namespace": "geo.SE"
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"format": "{street} {street-number}\n{postal-code} {locality}",
|
||||
"street": "{.locality as l}{l.street.name}",
|
||||
"street-number": [
|
||||
"{int(1,9)}",
|
||||
"{int(10,99)}",
|
||||
{ "format": "{int(100,999)}", "weight": 0.2 },
|
||||
{ "format": "{int(1,9)}{upper(1)}", "weight": 0.2 },
|
||||
{ "format": "{int(10,99)}{upper(1)}", "weight": 0.1 },
|
||||
{ "format": "{int(100,999)}{upper(1)}", "weight": 0.05 }
|
||||
],
|
||||
"postal-code": "{l.postal-code.code}",
|
||||
"locality": "{l.name}"
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
// Package geose is fejkdata's Swedish places module, whose categories are geo.SE.*.
|
||||
package geose
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"io/fs"
|
||||
)
|
||||
|
||||
// FS holds the module's categories and its manifest.
|
||||
//
|
||||
//go:embed *.json *.tsv .fejkdata.json
|
||||
var FS embed.FS
|
||||
|
||||
// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
|
||||
func Modules() []fs.FS { return []fs.FS{FS} }
|
||||
@@ -1,22 +0,0 @@
|
||||
package geose_test
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"testing"
|
||||
|
||||
"github.com/larvit/fejkdata"
|
||||
geose "github.com/larvit/fejkdata/data/geo/SE"
|
||||
)
|
||||
|
||||
func TestFSLoadsByItsManifest(t *testing.T) {
|
||||
if _, err := fs.Stat(geose.FS, ".fejkdata.json"); err != nil {
|
||||
t.Fatalf("FS holds no manifest: %v", err)
|
||||
}
|
||||
f, err := fejkdata.New(fejkdata.WithDataFS(geose.Modules()...), fejkdata.WithSeed(1))
|
||||
if err != nil {
|
||||
t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
|
||||
}
|
||||
if len(f.List()) == 0 {
|
||||
t.Error("List() is empty, want the module's categories")
|
||||
}
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
{ "format": "{name}", "rows": "locality.tsv", "key": "name", "parent": "municipality", "weight": "population" }
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1 +0,0 @@
|
||||
{ "format": "{name}", "rows": "municipality.tsv", "key": "code", "name": "name", "parent": "region", "weight": "population" }
|
||||
@@ -1,291 +0,0 @@
|
||||
code name region population
|
||||
0114 Upplands Väsby 01 50323
|
||||
0115 Vallentuna 01 35119
|
||||
0117 Österåker 01 49787
|
||||
0120 Värmdö 01 46635
|
||||
0123 Järfälla 01 88950
|
||||
0125 Ekerö 01 28910
|
||||
0126 Huddinge 01 114304
|
||||
0127 Botkyrka 01 95905
|
||||
0128 Salem 01 17507
|
||||
0136 Haninge 01 100895
|
||||
0138 Tyresö 01 49179
|
||||
0139 Upplands-Bro 01 32868
|
||||
0140 Nykvarn 01 12342
|
||||
0160 Täby 01 77744
|
||||
0162 Danderyd 01 32425
|
||||
0163 Sollentuna 01 77624
|
||||
0180 Stockholm 01 995574
|
||||
0181 Södertälje 01 102911
|
||||
0182 Nacka 01 112112
|
||||
0183 Sundbyberg 01 56274
|
||||
0184 Solna 01 85789
|
||||
0186 Lidingö 01 48377
|
||||
0187 Vaxholm 01 11822
|
||||
0188 Norrtälje 01 66585
|
||||
0191 Sigtuna 01 52767
|
||||
0192 Nynäshamn 01 30579
|
||||
0305 Håbo 03 22973
|
||||
0319 Älvkarleby 03 9552
|
||||
0330 Knivsta 03 21193
|
||||
0331 Heby 03 14345
|
||||
0360 Tierp 03 21104
|
||||
0380 Uppsala 03 248016
|
||||
0381 Enköping 03 48591
|
||||
0382 Östhammar 03 22138
|
||||
0428 Vingåker 04 8750
|
||||
0461 Gnesta 04 11458
|
||||
0480 Nyköping 04 58344
|
||||
0481 Oxelösund 04 12031
|
||||
0482 Flen 04 15362
|
||||
0483 Katrineholm 04 34154
|
||||
0484 Eskilstuna 04 107203
|
||||
0486 Strängnäs 04 39313
|
||||
0488 Trosa 04 14927
|
||||
0509 Ödeshög 05 5237
|
||||
0512 Ydre 05 3626
|
||||
0513 Kinda 05 9957
|
||||
0560 Boxholm 05 5517
|
||||
0561 Åtvidaberg 05 11467
|
||||
0562 Finspång 05 21623
|
||||
0563 Valdemarsvik 05 7525
|
||||
0580 Linköping 05 168035
|
||||
0581 Norrköping 05 144980
|
||||
0582 Söderköping 05 14789
|
||||
0583 Motala 05 43505
|
||||
0584 Vadstena 05 7490
|
||||
0586 Mjölby 05 28695
|
||||
0604 Aneby 06 6797
|
||||
0617 Gnosjö 06 9131
|
||||
0642 Mullsjö 06 7594
|
||||
0643 Habo 06 13456
|
||||
0662 Gislaved 06 28936
|
||||
0665 Vaggeryd 06 14825
|
||||
0680 Jönköping 06 147654
|
||||
0682 Nässjö 06 31587
|
||||
0683 Värnamo 06 34542
|
||||
0684 Sävsjö 06 11563
|
||||
0685 Vetlanda 06 27528
|
||||
0686 Eksjö 06 17792
|
||||
0687 Tranås 06 18604
|
||||
0760 Uppvidinge 07 9061
|
||||
0761 Lessebo 07 8289
|
||||
0763 Tingsryd 07 11966
|
||||
0764 Alvesta 07 19830
|
||||
0765 Älmhult 07 17653
|
||||
0767 Markaryd 07 9938
|
||||
0780 Växjö 07 98334
|
||||
0781 Ljungby 07 28280
|
||||
0821 Högsby 08 5321
|
||||
0834 Torsås 08 6984
|
||||
0840 Mörbylånga 08 16224
|
||||
0860 Hultsfred 08 13673
|
||||
0861 Mönsterås 08 13069
|
||||
0862 Emmaboda 08 9006
|
||||
0880 Kalmar 08 72704
|
||||
0881 Nybro 08 19951
|
||||
0882 Oskarshamn 08 26923
|
||||
0883 Västervik 08 36447
|
||||
0884 Vimmerby 08 15384
|
||||
0885 Borgholm 08 10666
|
||||
0980 Gotland 09 60971
|
||||
1060 Olofström 10 13000
|
||||
1080 Karlskrona 10 66301
|
||||
1081 Ronneby 10 28741
|
||||
1082 Karlshamn 10 31751
|
||||
1083 Sölvesborg 10 17430
|
||||
1214 Svalöv 12 14543
|
||||
1230 Staffanstorp 12 27303
|
||||
1231 Burlöv 12 20101
|
||||
1233 Vellinge 12 37816
|
||||
1256 Östra Göinge 12 13978
|
||||
1257 Örkelljunga 12 10277
|
||||
1260 Bjuv 12 15985
|
||||
1261 Kävlinge 12 32477
|
||||
1262 Lomma 12 24715
|
||||
1263 Svedala 12 23581
|
||||
1264 Skurup 12 17099
|
||||
1265 Sjöbo 12 19337
|
||||
1266 Hörby 12 15562
|
||||
1267 Höör 12 17518
|
||||
1270 Tomelilla 12 13639
|
||||
1272 Bromölla 12 12470
|
||||
1273 Osby 12 12947
|
||||
1275 Perstorp 12 7235
|
||||
1276 Klippan 12 17714
|
||||
1277 Åstorp 12 16449
|
||||
1278 Båstad 12 16026
|
||||
1280 Malmö 12 365644
|
||||
1281 Lund 12 131590
|
||||
1282 Landskrona 12 47309
|
||||
1283 Helsingborg 12 152091
|
||||
1284 Höganäs 12 28430
|
||||
1285 Eslöv 12 34922
|
||||
1286 Ystad 12 32106
|
||||
1287 Trelleborg 12 47269
|
||||
1290 Kristianstad 12 86379
|
||||
1291 Simrishamn 12 18890
|
||||
1292 Ängelholm 12 45110
|
||||
1293 Hässleholm 12 52114
|
||||
1315 Hylte 13 10196
|
||||
1380 Halmstad 13 106084
|
||||
1381 Laholm 13 26595
|
||||
1382 Falkenberg 13 47337
|
||||
1383 Varberg 13 69070
|
||||
1384 Kungsbacka 13 85792
|
||||
1401 Härryda 14 40003
|
||||
1402 Partille 14 41060
|
||||
1407 Öckerö 14 12771
|
||||
1415 Stenungsund 14 27851
|
||||
1419 Tjörn 14 16092
|
||||
1421 Orust 14 15352
|
||||
1427 Sotenäs 14 9104
|
||||
1430 Munkedal 14 10354
|
||||
1435 Tanum 14 12773
|
||||
1438 Dals-Ed 14 4606
|
||||
1439 Färgelanda 14 6376
|
||||
1440 Ale 14 32576
|
||||
1441 Lerum 14 43570
|
||||
1442 Vårgårda 14 12474
|
||||
1443 Bollebygd 14 9802
|
||||
1444 Grästorp 14 5555
|
||||
1445 Essunga 14 5560
|
||||
1446 Karlsborg 14 7023
|
||||
1447 Gullspång 14 5031
|
||||
1452 Tranemo 14 11839
|
||||
1460 Bengtsfors 14 9076
|
||||
1461 Mellerud 14 9052
|
||||
1462 Lilla Edet 14 14442
|
||||
1463 Mark 14 35155
|
||||
1465 Svenljunga 14 10747
|
||||
1466 Herrljunga 14 9497
|
||||
1470 Vara 14 16088
|
||||
1471 Götene 14 13286
|
||||
1472 Tibro 14 11338
|
||||
1473 Töreboda 14 9043
|
||||
1480 Göteborg 14 608993
|
||||
1481 Mölndal 14 71420
|
||||
1482 Kungälv 14 50313
|
||||
1484 Lysekil 14 13907
|
||||
1485 Uddevalla 14 57010
|
||||
1486 Strömstad 14 13482
|
||||
1487 Vänersborg 14 40041
|
||||
1488 Trollhättan 14 59003
|
||||
1489 Alingsås 14 42722
|
||||
1490 Borås 14 114872
|
||||
1491 Ulricehamn 14 24985
|
||||
1492 Åmål 14 11906
|
||||
1493 Mariestad 14 24583
|
||||
1494 Lidköping 14 40425
|
||||
1495 Skara 14 18707
|
||||
1496 Skövde 14 57995
|
||||
1497 Hjo 14 9350
|
||||
1498 Tidaholm 14 12805
|
||||
1499 Falköping 14 32806
|
||||
1715 Kil 17 12061
|
||||
1730 Eda 17 8412
|
||||
1737 Torsby 17 11321
|
||||
1760 Storfors 17 3788
|
||||
1761 Hammarö 17 16992
|
||||
1762 Munkfors 17 3625
|
||||
1763 Forshaga 17 11520
|
||||
1764 Grums 17 9004
|
||||
1765 Årjäng 17 9825
|
||||
1766 Sunne 17 13356
|
||||
1780 Karlstad 17 98084
|
||||
1781 Kristinehamn 17 23756
|
||||
1782 Filipstad 17 9776
|
||||
1783 Hagfors 17 11418
|
||||
1784 Arvika 17 25547
|
||||
1785 Säffle 17 14899
|
||||
1814 Lekeberg 18 8606
|
||||
1860 Laxå 18 5423
|
||||
1861 Hallsberg 18 16120
|
||||
1862 Degerfors 18 9278
|
||||
1863 Hällefors 18 6321
|
||||
1864 Ljusnarsberg 18 4369
|
||||
1880 Örebro 18 160140
|
||||
1881 Kumla 18 22681
|
||||
1882 Askersund 18 11477
|
||||
1883 Karlskoga 18 30180
|
||||
1884 Nora 18 10639
|
||||
1885 Lindesberg 18 23141
|
||||
1904 Skinnskatteberg 19 4256
|
||||
1907 Surahammar 19 9845
|
||||
1960 Kungsör 19 8694
|
||||
1961 Hallstahammar 19 16653
|
||||
1962 Norberg 19 5452
|
||||
1980 Västerås 19 160634
|
||||
1981 Sala 19 22843
|
||||
1982 Fagersta 19 13072
|
||||
1983 Köping 19 25729
|
||||
1984 Arboga 19 13980
|
||||
2021 Vansbro 20 6752
|
||||
2023 Malung-Sälen 20 10254
|
||||
2026 Gagnef 20 10384
|
||||
2029 Leksand 20 16137
|
||||
2031 Rättvik 20 10998
|
||||
2034 Orsa 20 6851
|
||||
2039 Älvdalen 20 6882
|
||||
2061 Smedjebacken 20 10823
|
||||
2062 Mora 20 20540
|
||||
2080 Falun 20 59945
|
||||
2081 Borlänge 20 51425
|
||||
2082 Säter 20 11223
|
||||
2083 Hedemora 20 15281
|
||||
2084 Avesta 20 22417
|
||||
2085 Ludvika 20 26634
|
||||
2101 Ockelbo 21 5715
|
||||
2104 Hofors 21 9281
|
||||
2121 Ovanåker 21 11341
|
||||
2132 Nordanstig 21 9262
|
||||
2161 Ljusdal 21 18445
|
||||
2180 Gävle 21 103838
|
||||
2181 Sandviken 21 38360
|
||||
2182 Söderhamn 21 24545
|
||||
2183 Bollnäs 21 26243
|
||||
2184 Hudiksvall 21 37528
|
||||
2260 Ånge 22 9044
|
||||
2262 Timrå 22 17521
|
||||
2280 Härnösand 22 24515
|
||||
2281 Sundsvall 22 99048
|
||||
2282 Kramfors 22 17491
|
||||
2283 Sollefteå 22 18396
|
||||
2284 Örnsköldsvik 22 55443
|
||||
2303 Ragunda 23 5146
|
||||
2305 Bräcke 23 6035
|
||||
2309 Krokom 23 15680
|
||||
2313 Strömsund 23 11023
|
||||
2321 Åre 23 12693
|
||||
2326 Berg 23 7108
|
||||
2361 Härjedalen 23 10175
|
||||
2380 Östersund 23 64979
|
||||
2401 Nordmaling 24 6942
|
||||
2403 Bjurholm 24 2359
|
||||
2404 Vindeln 24 5417
|
||||
2409 Robertsfors 24 6690
|
||||
2417 Norsjö 24 3968
|
||||
2418 Malå 24 2962
|
||||
2421 Storuman 24 5577
|
||||
2422 Sorsele 24 2357
|
||||
2425 Dorotea 24 2294
|
||||
2460 Vännäs 24 9132
|
||||
2462 Vilhelmina 24 6229
|
||||
2463 Åsele 24 2694
|
||||
2480 Umeå 24 134249
|
||||
2481 Lycksele 24 12118
|
||||
2482 Skellefteå 24 78150
|
||||
2505 Arvidsjaur 25 6089
|
||||
2506 Arjeplog 25 2599
|
||||
2510 Jokkmokk 25 4701
|
||||
2513 Överkalix 25 3201
|
||||
2514 Kalix 25 15391
|
||||
2518 Övertorneå 25 4057
|
||||
2521 Pajala 25 5857
|
||||
2523 Gällivare 25 17233
|
||||
2560 Älvsbyn 25 7774
|
||||
2580 Luleå 25 79645
|
||||
2581 Piteå 25 42447
|
||||
2582 Boden 25 28049
|
||||
2583 Haparanda 25 9151
|
||||
2584 Kiruna 25 22426
|
||||
|
@@ -1 +0,0 @@
|
||||
{ "format": "{code}", "rows": "postal-code.tsv", "key": "code", "parent": "locality" }
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1 +0,0 @@
|
||||
{ "format": "{name}", "rows": "region.tsv", "key": "code", "name": "name", "weight": "population" }
|
||||
@@ -1,22 +0,0 @@
|
||||
code name population timezone
|
||||
01 Stockholms län 2473307 Europe/Stockholm
|
||||
03 Uppsala län 407912 Europe/Stockholm
|
||||
04 Södermanlands län 301542 Europe/Stockholm
|
||||
05 Östergötlands län 472446 Europe/Stockholm
|
||||
06 Jönköpings län 370009 Europe/Stockholm
|
||||
07 Kronobergs län 203351 Europe/Stockholm
|
||||
08 Kalmar län 246352 Europe/Stockholm
|
||||
09 Gotlands län 60971 Europe/Stockholm
|
||||
10 Blekinge län 157223 Europe/Stockholm
|
||||
12 Skåne län 1428626 Europe/Stockholm
|
||||
13 Hallands län 345074 Europe/Stockholm
|
||||
14 Västra Götalands län 1772821 Europe/Stockholm
|
||||
17 Värmlands län 283384 Europe/Stockholm
|
||||
18 Örebro län 308375 Europe/Stockholm
|
||||
19 Västmanlands län 281158 Europe/Stockholm
|
||||
20 Dalarnas län 286546 Europe/Stockholm
|
||||
21 Gävleborgs län 284558 Europe/Stockholm
|
||||
22 Västernorrlands län 241458 Europe/Stockholm
|
||||
23 Jämtlands län 132839 Europe/Stockholm
|
||||
24 Västerbottens län 281138 Europe/Stockholm
|
||||
25 Norrbottens län 248620 Europe/Stockholm
|
||||
|
@@ -1 +0,0 @@
|
||||
{ "format": "{name}", "rows": "street.tsv", "parent": "locality", "weight": "segments" }
|
||||
-174317
File diff suppressed because it is too large
Load Diff
@@ -1,11 +0,0 @@
|
||||
{
|
||||
"index": {
|
||||
"geo.US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
|
||||
"geo.US.locality": {"parent":"municipality","paths":["","code","lat","lon","municipality","name","population","postal-code","postal-code.addresses","postal-code.code","postal-code.locality","street","street.addresses","street.locality","street.name"]},
|
||||
"geo.US.municipality": {"parent":"region","paths":["","code","locality","locality.code","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.addresses","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.addresses","locality.street.locality","locality.street.name","name","population","region"]},
|
||||
"geo.US.postal-code": {"parent":"locality","paths":["","addresses","code","locality"]},
|
||||
"geo.US.region": {"paths":["","abbr","code","municipality","municipality.code","municipality.locality","municipality.locality.code","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.addresses","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.addresses","municipality.locality.street.locality","municipality.locality.street.name","municipality.name","municipality.population","municipality.region","name","population","timezone"]},
|
||||
"geo.US.street": {"parent":"locality","paths":["","addresses","locality","name"]}
|
||||
},
|
||||
"namespace": "geo.US"
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
{
|
||||
"format": "{street-number} {street}\n{locality}, {region} {postal-code}",
|
||||
"street": "{.locality as l}{l.street.name}",
|
||||
"street-number": [
|
||||
"{int(10,99)}",
|
||||
{ "format": "{int(100,999)}", "weight": 2 },
|
||||
{ "format": "{int(1000,9999)}", "weight": 0.5 },
|
||||
{ "format": "{int(10,99)}{int(100,999)}", "weight": 0.3 }
|
||||
],
|
||||
"locality": "{l.name}",
|
||||
"region": "{l..municipality..region.abbr}",
|
||||
"postal-code": "{l.postal-code.code}"
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
// Package geous is fejkdata's US places module, whose categories are geo.US.*.
|
||||
package geous
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"io/fs"
|
||||
)
|
||||
|
||||
// FS holds the module's categories and its manifest.
|
||||
//
|
||||
//go:embed *.json *.tsv .fejkdata.json
|
||||
var FS embed.FS
|
||||
|
||||
// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
|
||||
func Modules() []fs.FS { return []fs.FS{FS} }
|
||||
@@ -1,22 +0,0 @@
|
||||
package geous_test
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"testing"
|
||||
|
||||
"github.com/larvit/fejkdata"
|
||||
geous "github.com/larvit/fejkdata/data/geo/US"
|
||||
)
|
||||
|
||||
func TestFSLoadsByItsManifest(t *testing.T) {
|
||||
if _, err := fs.Stat(geous.FS, ".fejkdata.json"); err != nil {
|
||||
t.Fatalf("FS holds no manifest: %v", err)
|
||||
}
|
||||
f, err := fejkdata.New(fejkdata.WithDataFS(geous.Modules()...), fejkdata.WithSeed(1))
|
||||
if err != nil {
|
||||
t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
|
||||
}
|
||||
if len(f.List()) == 0 {
|
||||
t.Error("List() is empty, want the module's categories")
|
||||
}
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
{ "format": "{name}", "rows": "locality.tsv", "key": "code", "name": "name", "parent": "municipality", "weight": "population" }
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1 +0,0 @@
|
||||
{ "format": "{name}", "rows": "municipality.tsv", "key": "code", "name": "name", "parent": "region", "weight": "population" }
|
||||
@@ -1,887 +0,0 @@
|
||||
code name region population
|
||||
01005 Barbour County AL 24607
|
||||
01015 Calhoun County AL 115834
|
||||
01021 Chilton County AL 47708
|
||||
01043 Cullman County AL 94009
|
||||
01045 Dale County AL 49912
|
||||
01049 DeKalb County AL 74085
|
||||
01051 Elmore County AL 91577
|
||||
01055 Etowah County AL 103886
|
||||
01065 Hale County AL 15194
|
||||
01069 Houston County AL 110318
|
||||
01071 Jackson County AL 54281
|
||||
01073 Jefferson County AL 665742
|
||||
01077 Lauderdale County AL 97135
|
||||
01081 Lee County AL 189881
|
||||
01089 Madison County AL 433516
|
||||
01095 Marshall County AL 103537
|
||||
01097 Mobile County AL 411658
|
||||
01101 Montgomery County AL 225891
|
||||
01103 Morgan County AL 126483
|
||||
01109 Pike County AL 33688
|
||||
01111 Randolph County AL 23163
|
||||
01113 Russell County AL 58898
|
||||
01115 St. Clair County AL 98206
|
||||
01117 Shelby County AL 238552
|
||||
01121 Talladega County AL 81603
|
||||
01125 Tuscaloosa County AL 241368
|
||||
01127 Walker County AL 65140
|
||||
02020 Anchorage Municipality AK 287155
|
||||
02110 Juneau City and Borough AK 31609
|
||||
02185 North Slope Borough AK 10582
|
||||
04003 Cochise County AZ 126332
|
||||
04005 Coconino County AZ 144368
|
||||
04013 Maricopa County AZ 4689558
|
||||
04015 Mohave County AZ 228102
|
||||
04017 Navajo County AZ 109946
|
||||
04019 Pima County AZ 1074685
|
||||
04021 Pinal County AZ 539380
|
||||
04025 Yavapai County AZ 252552
|
||||
04027 Yuma County AZ 224449
|
||||
05005 Baxter County AR 42982
|
||||
05007 Benton County AR 332554
|
||||
05019 Clark County AR 20938
|
||||
05031 Craighead County AR 116957
|
||||
05035 Crittenden County AR 46210
|
||||
05039 Dallas County AR 6011
|
||||
05045 Faulkner County AR 133979
|
||||
05071 Johnson County AR 26225
|
||||
05083 Logan County AR 21633
|
||||
05085 Lonoke County AR 76664
|
||||
05089 Marion County AR 17593
|
||||
05091 Miller County AR 42357
|
||||
05111 Poinsett County AR 22483
|
||||
05113 Polk County AR 19486
|
||||
05119 Pulaski County AR 404611
|
||||
05125 Saline County AR 133288
|
||||
05131 Sebastian County AR 130641
|
||||
05143 Washington County AR 271213
|
||||
05145 White County AR 80085
|
||||
06001 Alameda County CA 1636630
|
||||
06007 Butte County CA 209211
|
||||
06009 Calaveras County CA 46605
|
||||
06013 Contra Costa County CA 1170070
|
||||
06017 El Dorado County CA 192323
|
||||
06019 Fresno County CA 1035456
|
||||
06023 Humboldt County CA 131647
|
||||
06025 Imperial County CA 181411
|
||||
06029 Kern County CA 927068
|
||||
06031 Kings County CA 154327
|
||||
06035 Lassen County CA 28117
|
||||
06037 Los Angeles County CA 9694934
|
||||
06039 Madera County CA 167927
|
||||
06045 Mendocino County CA 88122
|
||||
06047 Merced County CA 297260
|
||||
06053 Monterey County CA 433729
|
||||
06055 Napa County CA 132949
|
||||
06057 Nevada County CA 101911
|
||||
06059 Orange County CA 3149507
|
||||
06061 Placer County CA 442081
|
||||
06065 Riverside County CA 2544916
|
||||
06067 Sacramento County CA 1618460
|
||||
06071 San Bernardino County CA 2224091
|
||||
06073 San Diego County CA 3282248
|
||||
06075 San Francisco County CA 826079
|
||||
06077 San Joaquin County CA 823815
|
||||
06079 San Luis Obispo County CA 282367
|
||||
06081 San Mateo County CA 743568
|
||||
06083 Santa Barbara County CA 442065
|
||||
06085 Santa Clara County CA 1914391
|
||||
06087 Santa Cruz County CA 258852
|
||||
06089 Shasta County CA 181648
|
||||
06095 Solano County CA 455376
|
||||
06097 Sonoma County CA 486444
|
||||
06099 Stanislaus County CA 557719
|
||||
06103 Tehama County CA 64665
|
||||
06107 Tulare County CA 485146
|
||||
06111 Ventura County CA 830851
|
||||
06113 Yolo County CA 224410
|
||||
06115 Yuba County CA 88691
|
||||
08001 Adams County CO 554668
|
||||
08005 Arapahoe County CO 673820
|
||||
08011 Bent County CO 5716
|
||||
08013 Boulder County CO 328560
|
||||
08029 Delta County CO 32178
|
||||
08031 Denver County CO 740613
|
||||
08033 Dolores County CO 2466
|
||||
08035 Douglas County CO 399396
|
||||
08037 Eagle County CO 54291
|
||||
08041 El Paso County CO 757040
|
||||
08043 Fremont County CO 50039
|
||||
08045 Garfield County CO 63474
|
||||
08049 Grand County CO 16496
|
||||
08051 Gunnison County CO 17439
|
||||
08059 Jefferson County CO 580451
|
||||
08067 La Plata County CO 56898
|
||||
08069 Larimer County CO 377292
|
||||
08089 Otero County CO 17683
|
||||
08091 Ouray County CO 5265
|
||||
08101 Pueblo County CO 169277
|
||||
08105 Rio Grande County CO 11087
|
||||
08107 Routt County CO 25329
|
||||
08123 Weld County CO 378426
|
||||
09110 Capitol Planning Region CT 994115
|
||||
09120 Greater Bridgeport Planning Region CT 337697
|
||||
09140 Naugatuck Valley Planning Region CT 463349
|
||||
09150 Northeastern Connecticut Planning Region CT 98096
|
||||
09170 South Central Connecticut Planning Region CT 578741
|
||||
09180 Southeastern Connecticut Planning Region CT 284015
|
||||
09190 Western Connecticut Planning Region CT 640482
|
||||
10001 Kent County DE 194786
|
||||
10003 New Castle County DE 588026
|
||||
10005 Sussex County DE 277140
|
||||
11001 District of Columbia DC 693645
|
||||
12001 Alachua County FL 290028
|
||||
12005 Bay County FL 204479
|
||||
12009 Brevard County FL 663982
|
||||
12011 Broward County FL 2013317
|
||||
12017 Citrus County FL 171666
|
||||
12031 Duval County FL 1062963
|
||||
12033 Escambia County FL 333834
|
||||
12035 Flagler County FL 140360
|
||||
12045 Gulf County FL 15943
|
||||
12057 Hillsborough County FL 1574115
|
||||
12069 Lake County FL 456068
|
||||
12071 Lee County FL 875607
|
||||
12073 Leon County FL 299048
|
||||
12081 Manatee County FL 468200
|
||||
12085 Martin County FL 166272
|
||||
12086 Miami-Dade County FL 2802029
|
||||
12087 Monroe County FL 80406
|
||||
12089 Nassau County FL 106879
|
||||
12091 Okaloosa County FL 221810
|
||||
12095 Orange County FL 1528002
|
||||
12097 Osceola County FL 481718
|
||||
12099 Palm Beach County FL 1575726
|
||||
12101 Pasco County FL 674516
|
||||
12103 Pinellas County FL 948563
|
||||
12105 Polk County FL 874790
|
||||
12107 Putnam County FL 77734
|
||||
12109 St. Johns County FL 346328
|
||||
12111 St. Lucie County FL 402449
|
||||
12115 Sarasota County FL 479958
|
||||
12117 Seminole County FL 491884
|
||||
12125 Union County FL 16250
|
||||
12127 Volusia County FL 606573
|
||||
13021 Bibb County GA 157556
|
||||
13029 Bryan County GA 52062
|
||||
13031 Bulloch County GA 86949
|
||||
13039 Camden County GA 60143
|
||||
13045 Carroll County GA 131036
|
||||
13051 Chatham County GA 311855
|
||||
13057 Cherokee County GA 299273
|
||||
13059 Clarke County GA 129921
|
||||
13063 Clayton County GA 297471
|
||||
13067 Cobb County GA 793345
|
||||
13073 Columbia County GA 169189
|
||||
13077 Coweta County GA 160240
|
||||
13089 DeKalb County GA 774394
|
||||
13095 Dougherty County GA 82616
|
||||
13113 Fayette County GA 125156
|
||||
13121 Fulton County GA 1098791
|
||||
13133 Greene County GA 21367
|
||||
13135 Gwinnett County GA 1018099
|
||||
13137 Habersham County GA 50416
|
||||
13139 Hall County GA 226568
|
||||
13151 Henry County GA 264922
|
||||
13153 Houston County GA 178214
|
||||
13157 Jackson County GA 99265
|
||||
13179 Liberty County GA 70313
|
||||
13185 Lowndes County GA 122867
|
||||
13215 Muscogee County GA 202171
|
||||
13217 Newton County GA 125583
|
||||
13237 Putnam County GA 23722
|
||||
13245 Richmond County GA 206559
|
||||
13247 Rockdale County GA 98416
|
||||
13255 Spalding County GA 70775
|
||||
13277 Tift County GA 42254
|
||||
13297 Walton County GA 112696
|
||||
13299 Ware County GA 36906
|
||||
16001 Ada County ID 546141
|
||||
16019 Bonneville County ID 135771
|
||||
16027 Canyon County ID 275123
|
||||
16047 Gooding County ID 16446
|
||||
16057 Latah County ID 41842
|
||||
16083 Twin Falls County ID 97539
|
||||
17001 Adams County IL 64267
|
||||
17013 Calhoun County IL 4300
|
||||
17019 Champaign County IL 209972
|
||||
17029 Coles County IL 46607
|
||||
17031 Cook County IL 5194625
|
||||
17037 DeKalb County IL 101835
|
||||
17043 DuPage County IL 934298
|
||||
17045 Edgar County IL 16323
|
||||
17051 Fayette County IL 20790
|
||||
17053 Ford County IL 13216
|
||||
17055 Franklin County IL 36916
|
||||
17063 Grundy County IL 54052
|
||||
17071 Henderson County IL 5984
|
||||
17073 Henry County IL 48095
|
||||
17077 Jackson County IL 52501
|
||||
17085 Jo Daviess County IL 21594
|
||||
17089 Kane County IL 525757
|
||||
17093 Kendall County IL 145470
|
||||
17097 Lake County IL 719339
|
||||
17103 Lee County IL 32914
|
||||
17111 McHenry County IL 317751
|
||||
17113 McLean County IL 171419
|
||||
17119 Madison County IL 263110
|
||||
17139 Moultrie County IL 14371
|
||||
17143 Peoria County IL 178553
|
||||
17157 Randolph County IL 29921
|
||||
17161 Rock Island County IL 141869
|
||||
17163 St. Clair County IL 250708
|
||||
17167 Sangamon County IL 194170
|
||||
17169 Schuyler County IL 6657
|
||||
17177 Stephenson County IL 43220
|
||||
17195 Whiteside County IL 54121
|
||||
17197 Will County IL 712253
|
||||
17199 Williamson County IL 67225
|
||||
17201 Winnebago County IL 283674
|
||||
18001 Adams County IN 36650
|
||||
18003 Allen County IN 402329
|
||||
18005 Bartholomew County IN 85729
|
||||
18011 Boone County IN 80689
|
||||
18019 Clark County IN 130451
|
||||
18023 Clinton County IN 33322
|
||||
18027 Daviess County IN 34209
|
||||
18029 Dearborn County IN 51609
|
||||
18031 Decatur County IN 26576
|
||||
18033 DeKalb County IN 44535
|
||||
18035 Delaware County IN 113106
|
||||
18037 Dubois County IN 44016
|
||||
18039 Elkhart County IN 208774
|
||||
18043 Floyd County IN 82153
|
||||
18045 Fountain County IN 17013
|
||||
18049 Fulton County IN 20091
|
||||
18053 Grant County IN 66524
|
||||
18057 Hamilton County IN 387036
|
||||
18059 Hancock County IN 90969
|
||||
18063 Hendricks County IN 193510
|
||||
18069 Huntington County IN 37224
|
||||
18081 Johnson County IN 174262
|
||||
18089 Lake County IN 504612
|
||||
18091 LaPorte County IN 111294
|
||||
18095 Madison County IN 135088
|
||||
18097 Marion County IN 992196
|
||||
18105 Monroe County IN 143345
|
||||
18107 Montgomery County IN 38954
|
||||
18109 Morgan County IN 74967
|
||||
18127 Porter County IN 176049
|
||||
18141 St. Joseph County IN 272861
|
||||
18157 Tippecanoe County IN 190456
|
||||
18163 Vanderburgh County IN 181995
|
||||
18167 Vigo County IN 106512
|
||||
18169 Wabash County IN 30713
|
||||
18177 Wayne County IN 66169
|
||||
19009 Audubon County IA 5509
|
||||
19027 Carroll County IA 20303
|
||||
19055 Delaware County IA 17674
|
||||
19061 Dubuque County IA 99381
|
||||
19083 Hardin County IA 16455
|
||||
19095 Iowa County IA 16504
|
||||
19103 Johnson County IA 160044
|
||||
19113 Linn County IA 232028
|
||||
19123 Mahaska County IA 21880
|
||||
19127 Marshall County IA 39890
|
||||
19153 Polk County IA 516546
|
||||
19155 Pottawattamie County IA 92996
|
||||
19163 Scott County IA 175259
|
||||
19169 Story County IA 101291
|
||||
19179 Wapello County IA 35210
|
||||
19193 Woodbury County IA 106649
|
||||
20015 Butler County KS 69484
|
||||
20035 Cowley County KS 34382
|
||||
20041 Dickinson County KS 18637
|
||||
20045 Douglas County KS 120920
|
||||
20057 Ford County KS 33993
|
||||
20061 Geary County KS 36338
|
||||
20091 Johnson County KS 636906
|
||||
20099 Labette County KS 19685
|
||||
20103 Leavenworth County KS 84590
|
||||
20107 Linn County KS 9950
|
||||
20111 Lyon County KS 32266
|
||||
20113 McPherson County KS 30064
|
||||
20121 Miami County KS 36240
|
||||
20161 Riley County KS 72598
|
||||
20169 Saline County KS 53377
|
||||
20173 Sedgwick County KS 538433
|
||||
20175 Seward County KS 21073
|
||||
20177 Shawnee County KS 178607
|
||||
20209 Wyandotte County KS 170597
|
||||
21009 Barren County KY 45641
|
||||
21015 Boone County KY 145316
|
||||
21021 Boyle County KY 31632
|
||||
21029 Bullitt County KY 86454
|
||||
21035 Calloway County KY 39421
|
||||
21037 Campbell County KY 95441
|
||||
21047 Christian County KY 70115
|
||||
21049 Clark County KY 37888
|
||||
21059 Daviess County KY 104898
|
||||
21067 Fayette County KY 329751
|
||||
21073 Franklin County KY 52649
|
||||
21085 Grayson County KY 27406
|
||||
21093 Hardin County KY 113482
|
||||
21101 Henderson County KY 44255
|
||||
21111 Jefferson County KY 795222
|
||||
21117 Kenton County KY 175779
|
||||
21145 McCracken County KY 67553
|
||||
21151 Madison County KY 101696
|
||||
21177 Muhlenberg County KY 30634
|
||||
21183 Ohio County KY 23934
|
||||
21207 Russell County KY 18608
|
||||
21209 Scott County KY 62262
|
||||
21211 Shelby County KY 51243
|
||||
21227 Warren County KY 149375
|
||||
22009 Avoyelles Parish LA 37941
|
||||
22017 Caddo Parish LA 224226
|
||||
22019 Calcasieu Parish LA 208466
|
||||
22025 Catahoula Parish LA 8223
|
||||
22033 East Baton Rouge Parish LA 456180
|
||||
22049 Jackson Parish LA 15834
|
||||
22055 Lafayette Parish LA 257949
|
||||
22057 Lafourche Parish LA 95505
|
||||
22061 Lincoln Parish LA 48360
|
||||
22071 Orleans Parish LA 362154
|
||||
22073 Ouachita Parish LA 158542
|
||||
22077 Pointe Coupee Parish LA 19520
|
||||
22097 St. Landry Parish LA 80765
|
||||
22103 St. Tammany Parish LA 279108
|
||||
22105 Tangipahoa Parish LA 141346
|
||||
22117 Washington Parish LA 44937
|
||||
23001 Androscoggin County ME 116487
|
||||
23011 Kennebec County ME 129067
|
||||
23019 Penobscot County ME 157967
|
||||
23031 York County ME 222434
|
||||
24001 Allegany County MD 66848
|
||||
24003 Anne Arundel County MD 603380
|
||||
24015 Cecil County MD 107131
|
||||
24021 Frederick County MD 302883
|
||||
24031 Montgomery County MD 1074582
|
||||
24033 Prince George's County MD 970374
|
||||
24043 Washington County MD 157731
|
||||
24510 Baltimore city MD 569997
|
||||
25001 Barnstable County MA 233539
|
||||
25005 Bristol County MA 593640
|
||||
25009 Essex County MA 826653
|
||||
25013 Hampden County MA 464338
|
||||
25015 Hampshire County MA 164065
|
||||
25017 Middlesex County MA 1669979
|
||||
25021 Norfolk County MA 739749
|
||||
25023 Plymouth County MA 546829
|
||||
25025 Suffolk County MA 791891
|
||||
25027 Worcester County MA 888502
|
||||
26003 Alger County MI 8641
|
||||
26021 Berrien County MI 152444
|
||||
26037 Clinton County MI 80432
|
||||
26065 Ingham County MI 289709
|
||||
26077 Kalamazoo County MI 263795
|
||||
26081 Kent County MI 675232
|
||||
26099 Macomb County MI 886221
|
||||
26121 Muskegon County MI 177901
|
||||
26125 Oakland County MI 1288337
|
||||
26139 Ottawa County MI 308459
|
||||
26161 Washtenaw County MI 370214
|
||||
26163 Wayne County MI 1769038
|
||||
27003 Anoka County MN 381605
|
||||
27013 Blue Earth County MN 70634
|
||||
27017 Carlton County MN 36711
|
||||
27035 Crow Wing County MN 69132
|
||||
27037 Dakota County MN 457710
|
||||
27053 Hennepin County MN 1284784
|
||||
27061 Itasca County MN 45404
|
||||
27067 Kandiyohi County MN 44720
|
||||
27071 Koochiching County MN 11587
|
||||
27083 Lyon County MN 25684
|
||||
27109 Olmsted County MN 166731
|
||||
27111 Otter Tail County MN 61041
|
||||
27115 Pine County MN 30472
|
||||
27123 Ramsey County MN 541623
|
||||
27127 Redwood County MN 15265
|
||||
27131 Rice County MN 69939
|
||||
27137 St. Louis County MN 200518
|
||||
27139 Scott County MN 159017
|
||||
27141 Sherburne County MN 104194
|
||||
27143 Sibley County MN 15356
|
||||
27145 Stearns County MN 164110
|
||||
27147 Steele County MN 37464
|
||||
27163 Washington County MN 286895
|
||||
27165 Watonwan County MN 11459
|
||||
27167 Wilkin County MN 6288
|
||||
27171 Wright County MN 157559
|
||||
28003 Alcorn County MS 34569
|
||||
28033 DeSoto County MS 197918
|
||||
28035 Forrest County MS 79034
|
||||
28045 Hancock County MS 46873
|
||||
28047 Harrison County MS 217136
|
||||
28049 Hinds County MS 211888
|
||||
28059 Jackson County MS 147666
|
||||
28071 Lafayette County MS 59597
|
||||
28073 Lamar County MS 67403
|
||||
28089 Madison County MS 116298
|
||||
28095 Monroe County MS 33318
|
||||
28113 Pike County MS 38712
|
||||
28117 Prentiss County MS 25284
|
||||
28133 Sunflower County MS 22893
|
||||
28151 Washington County MS 40446
|
||||
28159 Winston County MS 17473
|
||||
29001 Adair County MO 25684
|
||||
29003 Andrew County MO 18241
|
||||
29019 Boone County MO 191746
|
||||
29031 Cape Girardeau County MO 83999
|
||||
29037 Cass County MO 115859
|
||||
29051 Cole County MO 77908
|
||||
29069 Dunklin County MO 26855
|
||||
29077 Greene County MO 309286
|
||||
29095 Jackson County MO 732994
|
||||
29097 Jasper County MO 127428
|
||||
29099 Jefferson County MO 233132
|
||||
29105 Laclede County MO 36943
|
||||
29115 Linn County MO 11815
|
||||
29123 Madison County MO 12858
|
||||
29127 Marion County MO 28525
|
||||
29155 Pemiscot County MO 14465
|
||||
29161 Phelps County MO 45247
|
||||
29167 Polk County MO 33604
|
||||
29169 Pulaski County MO 54442
|
||||
29175 Randolph County MO 24513
|
||||
29183 St. Charles County MO 426499
|
||||
29187 St. Francois County MO 67809
|
||||
29189 St. Louis County MO 990911
|
||||
29195 Saline County MO 23265
|
||||
29510 St. Louis city MO 278144
|
||||
30013 Cascade County MT 85029
|
||||
30017 Custer County MT 12034
|
||||
30031 Gallatin County MT 128740
|
||||
30049 Lewis and Clark County MT 75331
|
||||
30057 Madison County MT 10026
|
||||
30063 Missoula County MT 123513
|
||||
30081 Ravalli County MT 48582
|
||||
30111 Yellowstone County MT 172692
|
||||
31001 Adams County NE 31071
|
||||
31023 Butler County NE 8507
|
||||
31053 Dodge County NE 38057
|
||||
31055 Douglas County NE 606460
|
||||
31067 Gage County NE 21711
|
||||
31093 Howard County NE 6538
|
||||
31109 Lancaster County NE 334049
|
||||
31111 Lincoln County NE 33303
|
||||
31147 Richardson County NE 7530
|
||||
31151 Saline County NE 14996
|
||||
31181 Webster County NE 3336
|
||||
31185 York County NE 14564
|
||||
32003 Clark County NV 2407226
|
||||
32031 Washoe County NV 509386
|
||||
32510 Carson City NV 58571
|
||||
33007 Coos County NH 31343
|
||||
33011 Hillsborough County NH 433415
|
||||
33013 Merrimack County NH 158078
|
||||
33015 Rockingham County NH 324077
|
||||
34001 Atlantic County NJ 278657
|
||||
34003 Bergen County NJ 977026
|
||||
34007 Camden County NJ 535799
|
||||
34009 Cape May County NJ 93390
|
||||
34011 Cumberland County NJ 157148
|
||||
34013 Essex County NJ 896379
|
||||
34015 Gloucester County NJ 312638
|
||||
34017 Hudson County NJ 735033
|
||||
34021 Mercer County NJ 399289
|
||||
34023 Middlesex County NJ 883335
|
||||
34025 Monmouth County NJ 651035
|
||||
34027 Morris County NJ 524251
|
||||
34029 Ocean County NJ 673746
|
||||
34031 Passaic County NJ 531624
|
||||
34035 Somerset County NJ 356486
|
||||
34037 Sussex County NJ 148063
|
||||
34039 Union County NJ 601863
|
||||
35001 Bernalillo County NM 667601
|
||||
35005 Chaves County NM 63364
|
||||
35009 Curry County NM 46655
|
||||
35013 Doña Ana County NM 229091
|
||||
35029 Luna County NM 25407
|
||||
35035 Otero County NM 70368
|
||||
35041 Roosevelt County NM 18615
|
||||
35043 Sandoval County NM 159565
|
||||
35045 San Juan County NM 120340
|
||||
35047 San Miguel County NM 26260
|
||||
35049 Santa Fe County NM 156907
|
||||
35057 Torrance County NM 16100
|
||||
35061 Valencia County NM 82013
|
||||
36001 Albany County NY 321225
|
||||
36007 Broome County NY 195736
|
||||
36011 Cayuga County NY 74365
|
||||
36013 Chautauqua County NY 124126
|
||||
36015 Chemung County NY 80415
|
||||
36027 Dutchess County NY 300708
|
||||
36029 Erie County NY 946741
|
||||
36035 Fulton County NY 52216
|
||||
36039 Greene County NY 47238
|
||||
36045 Jefferson County NY 111540
|
||||
36047 Kings County NY 2653963
|
||||
36051 Livingston County NY 61438
|
||||
36053 Madison County NY 67120
|
||||
36055 Monroe County NY 750506
|
||||
36057 Montgomery County NY 50046
|
||||
36059 Nassau County NY 1398939
|
||||
36063 Niagara County NY 208912
|
||||
36065 Oneida County NY 226392
|
||||
36067 Onondaga County NY 466584
|
||||
36071 Orange County NY 417669
|
||||
36073 Orleans County NY 39825
|
||||
36093 Schenectady County NY 162581
|
||||
36101 Steuben County NY 91855
|
||||
36103 Suffolk County NY 1546090
|
||||
36105 Sullivan County NY 80586
|
||||
36107 Tioga County NY 47453
|
||||
36111 Ulster County NY 183330
|
||||
36119 Westchester County NY 1015743
|
||||
37013 Beaufort County NC 44670
|
||||
37017 Bladen County NC 30007
|
||||
37019 Brunswick County NC 174702
|
||||
37021 Buncombe County NC 277417
|
||||
37025 Cabarrus County NC 249725
|
||||
37027 Caldwell County NC 81105
|
||||
37031 Carteret County NC 70469
|
||||
37035 Catawba County NC 170172
|
||||
37045 Cleveland County NC 103325
|
||||
37049 Craven County NC 105025
|
||||
37051 Cumberland County NC 338473
|
||||
37057 Davidson County NC 180182
|
||||
37059 Davie County NC 45855
|
||||
37063 Durham County NC 347240
|
||||
37067 Forsyth County NC 401718
|
||||
37071 Gaston County NC 246558
|
||||
37081 Guilford County NC 562234
|
||||
37089 Henderson County NC 122375
|
||||
37097 Iredell County NC 211798
|
||||
37101 Johnston County NC 256448
|
||||
37107 Lenoir County NC 55837
|
||||
37119 Mecklenburg County NC 1233383
|
||||
37123 Montgomery County NC 26403
|
||||
37127 Nash County NC 99365
|
||||
37129 New Hanover County NC 245959
|
||||
37133 Onslow County NC 217175
|
||||
37135 Orange County NC 152498
|
||||
37137 Pamlico County NC 12758
|
||||
37153 Richmond County NC 42041
|
||||
37159 Rowan County NC 155096
|
||||
37161 Rutherford County NC 65745
|
||||
37165 Scotland County NC 33322
|
||||
37179 Union County NC 267674
|
||||
37183 Wake County NC 1257235
|
||||
37191 Wayne County NC 122278
|
||||
37195 Wilson County NC 81150
|
||||
38009 Bottineau County ND 6284
|
||||
38015 Burleigh County ND 103251
|
||||
38017 Cass County ND 201794
|
||||
38035 Grand Forks County ND 74501
|
||||
38101 Ward County ND 68233
|
||||
38105 Williams County ND 41767
|
||||
39003 Allen County OH 100881
|
||||
39009 Athens County OH 63197
|
||||
39017 Butler County OH 400128
|
||||
39023 Clark County OH 135340
|
||||
39029 Columbiana County OH 99377
|
||||
39035 Cuyahoga County OH 1232925
|
||||
39041 Delaware County OH 242032
|
||||
39043 Erie County OH 73296
|
||||
39045 Fairfield County OH 169752
|
||||
39047 Fayette County OH 28552
|
||||
39049 Franklin County OH 1361536
|
||||
39051 Fulton County OH 41853
|
||||
39061 Hamilton County OH 838418
|
||||
39063 Hancock County OH 75034
|
||||
39065 Hardin County OH 30072
|
||||
39075 Holmes County OH 44970
|
||||
39077 Huron County OH 58250
|
||||
39079 Jackson County OH 32817
|
||||
39083 Knox County OH 64002
|
||||
39085 Lake County OH 232217
|
||||
39089 Licking County OH 185564
|
||||
39093 Lorain County OH 323219
|
||||
39095 Lucas County OH 423347
|
||||
39099 Mahoning County OH 224706
|
||||
39101 Marion County OH 65115
|
||||
39103 Medina County OH 185025
|
||||
39113 Montgomery County OH 539598
|
||||
39131 Pike County OH 27061
|
||||
39133 Portage County OH 163404
|
||||
39135 Preble County OH 40664
|
||||
39139 Richland County OH 124893
|
||||
39149 Shelby County OH 48065
|
||||
39151 Stark County OH 373771
|
||||
39153 Summit County OH 538376
|
||||
39155 Trumbull County OH 198972
|
||||
39159 Union County OH 73446
|
||||
39165 Warren County OH 257181
|
||||
39169 Wayne County OH 116758
|
||||
39171 Williams County OH 36549
|
||||
39173 Wood County OH 134176
|
||||
40001 Adair County OK 19828
|
||||
40017 Canadian County OK 187189
|
||||
40027 Cleveland County OK 303973
|
||||
40031 Comanche County OK 122158
|
||||
40037 Creek County OK 74967
|
||||
40039 Custer County OK 28175
|
||||
40047 Garfield County OK 61779
|
||||
40087 McClain County OK 49647
|
||||
40093 Major County OK 7536
|
||||
40101 Muskogee County OK 66708
|
||||
40109 Oklahoma County OK 822125
|
||||
40119 Payne County OK 83889
|
||||
40121 Pittsburg County OK 43320
|
||||
40125 Pottawatomie County OK 75102
|
||||
40143 Tulsa County OK 698782
|
||||
40145 Wagoner County OK 93263
|
||||
41005 Clackamas County OR 426280
|
||||
41009 Columbia County OR 54091
|
||||
41017 Deschutes County OR 213072
|
||||
41029 Jackson County OR 221795
|
||||
41033 Josephine County OR 87867
|
||||
41035 Klamath County OR 70274
|
||||
41039 Lane County OR 381584
|
||||
41041 Lincoln County OR 50636
|
||||
41047 Marion County OR 355777
|
||||
41049 Morrow County OR 12602
|
||||
41051 Multnomah County OR 795391
|
||||
41067 Washington County OR 611708
|
||||
41071 Yamhill County OR 110024
|
||||
42003 Allegheny County PA 1225035
|
||||
42011 Berks County PA 440072
|
||||
42017 Bucks County PA 647828
|
||||
42019 Butler County PA 200169
|
||||
42021 Cambria County PA 128968
|
||||
42035 Clinton County PA 37870
|
||||
42037 Columbia County PA 66193
|
||||
42043 Dauphin County PA 293351
|
||||
42045 Delaware County PA 580937
|
||||
42049 Erie County PA 265832
|
||||
42055 Franklin County PA 160652
|
||||
42061 Huntingdon County PA 43001
|
||||
42063 Indiana County PA 82878
|
||||
42069 Lackawanna County PA 216502
|
||||
42071 Lancaster County PA 563159
|
||||
42073 Lawrence County PA 83911
|
||||
42075 Lebanon County PA 146380
|
||||
42077 Lehigh County PA 384383
|
||||
42079 Luzerne County PA 332126
|
||||
42081 Lycoming County PA 112587
|
||||
42087 Mifflin County PA 46127
|
||||
42091 Montgomery County PA 877643
|
||||
42095 Northampton County PA 324411
|
||||
42097 Northumberland County PA 89920
|
||||
42101 Philadelphia County PA 1574281
|
||||
42107 Schuylkill County PA 145085
|
||||
42111 Somerset County PA 72004
|
||||
42119 Union County PA 42313
|
||||
42121 Venango County PA 49346
|
||||
42127 Wayne County PA 51703
|
||||
42129 Westmoreland County PA 349324
|
||||
44005 Newport County RI 83051
|
||||
44007 Providence County RI 678179
|
||||
45003 Aiken County SC 181515
|
||||
45005 Allendale County SC 7355
|
||||
45007 Anderson County SC 219930
|
||||
45011 Barnwell County SC 20653
|
||||
45015 Berkeley County SC 274666
|
||||
45019 Charleston County SC 436200
|
||||
45023 Chester County SC 33001
|
||||
45041 Florence County SC 138504
|
||||
45045 Greenville County SC 583125
|
||||
45051 Horry County SC 427551
|
||||
45059 Laurens County SC 71848
|
||||
45077 Pickens County SC 139198
|
||||
45079 Richland County SC 434956
|
||||
45083 Spartanburg County SC 380857
|
||||
46005 Beadle County SD 19624
|
||||
46065 Hughes County SD 17570
|
||||
46081 Lawrence County SD 29459
|
||||
46099 Minnehaha County SD 212691
|
||||
46103 Pennington County SD 116792
|
||||
46127 Union County SD 17402
|
||||
47009 Blount County TN 143820
|
||||
47011 Bradley County TN 115465
|
||||
47017 Carroll County TN 29200
|
||||
47021 Cheatham County TN 42778
|
||||
47037 Davidson County TN 745904
|
||||
47063 Hamblen County TN 68843
|
||||
47065 Hamilton County TN 390833
|
||||
47071 Hardin County TN 27983
|
||||
47073 Hawkins County TN 59559
|
||||
47083 Houston County TN 8479
|
||||
47093 Knox County TN 511453
|
||||
47113 Madison County TN 100790
|
||||
47125 Montgomery County TN 249935
|
||||
47127 Moore County TN 6920
|
||||
47141 Putnam County TN 86612
|
||||
47145 Roane County TN 57395
|
||||
47149 Rutherford County TN 386352
|
||||
47155 Sevier County TN 101342
|
||||
47157 Shelby County TN 910226
|
||||
47163 Sullivan County TN 163759
|
||||
47165 Sumner County TN 215538
|
||||
47167 Tipton County TN 62287
|
||||
47177 Warren County TN 43717
|
||||
47179 Washington County TN 141199
|
||||
47187 Williamson County TN 272061
|
||||
47189 Wilson County TN 175033
|
||||
48005 Angelina County TX 88154
|
||||
48023 Baylor County TX 3472
|
||||
48027 Bell County TX 402248
|
||||
48029 Bexar County TX 2160088
|
||||
48039 Brazoria County TX 419080
|
||||
48041 Brazos County TX 249088
|
||||
48053 Burnet County TX 57015
|
||||
48057 Calhoun County TX 19599
|
||||
48059 Callahan County TX 14491
|
||||
48061 Cameron County TX 433946
|
||||
48073 Cherokee County TX 53337
|
||||
48085 Collin County TX 1297179
|
||||
48089 Colorado County TX 21439
|
||||
48091 Comal County TX 209166
|
||||
48111 Dallam County TX 7508
|
||||
48113 Dallas County TX 2661397
|
||||
48121 Denton County TX 1069346
|
||||
48135 Ector County TX 173801
|
||||
48139 Ellis County TX 240867
|
||||
48141 El Paso County TX 877858
|
||||
48149 Fayette County TX 25459
|
||||
48157 Fort Bend County TX 975191
|
||||
48165 Gaines County TX 23956
|
||||
48167 Galveston County TX 372207
|
||||
48171 Gillespie County TX 28527
|
||||
48177 Gonzales County TX 20159
|
||||
48181 Grayson County TX 153613
|
||||
48183 Gregg County TX 126095
|
||||
48187 Guadalupe County TX 201111
|
||||
48193 Hamilton County TX 8680
|
||||
48201 Harris County TX 5045026
|
||||
48209 Hays County TX 304390
|
||||
48215 Hidalgo County TX 921549
|
||||
48231 Hunt County TX 123336
|
||||
48245 Jefferson County TX 254321
|
||||
48251 Johnson County TX 218048
|
||||
48257 Kaufman County TX 209235
|
||||
48273 Kleberg County TX 30315
|
||||
48277 Lamar County TX 51503
|
||||
48291 Liberty County TX 121364
|
||||
48293 Limestone County TX 22849
|
||||
48303 Lubbock County TX 328906
|
||||
48307 McCulloch County TX 7472
|
||||
48309 McLennan County TX 272020
|
||||
48325 Medina County TX 56881
|
||||
48329 Midland County TX 187855
|
||||
48331 Milam County TX 26258
|
||||
48339 Montgomery County TX 781194
|
||||
48349 Navarro County TX 57181
|
||||
48355 Nueces County TX 352992
|
||||
48361 Orange County TX 86266
|
||||
48381 Randall County TX 152351
|
||||
48397 Rockwall County TX 140738
|
||||
48403 Sabine County TX 10012
|
||||
48411 San Saba County TX 5787
|
||||
48417 Shackelford County TX 3197
|
||||
48423 Smith County TX 252549
|
||||
48439 Tarrant County TX 2248466
|
||||
48441 Taylor County TX 150077
|
||||
48451 Tom Green County TX 120602
|
||||
48453 Travis County TX 1389670
|
||||
48465 Val Verde County TX 47835
|
||||
48479 Webb County TX 281224
|
||||
48485 Wichita County TX 129555
|
||||
48491 Williamson County TX 752827
|
||||
48497 Wise County TX 83778
|
||||
49005 Cache County UT 145000
|
||||
49007 Carbon County UT 20680
|
||||
49011 Davis County UT 381227
|
||||
49021 Iron County UT 67141
|
||||
49023 Juab County UT 13567
|
||||
49027 Millard County UT 14057
|
||||
49035 Salt Lake County UT 1220916
|
||||
49037 San Juan County UT 14542
|
||||
49039 Sanpete County UT 31453
|
||||
49041 Sevier County UT 22827
|
||||
49049 Utah County UT 759859
|
||||
49053 Washington County UT 213670
|
||||
49057 Weber County UT 278174
|
||||
50007 Chittenden County VT 169115
|
||||
50023 Washington County VT 59652
|
||||
51035 Carroll County VA 29299
|
||||
51093 Isle of Wight County VA 41321
|
||||
51107 Loudoun County VA 449749
|
||||
51135 Nottoway County VA 15796
|
||||
51183 Sussex County VA 10755
|
||||
51195 Wise County VA 34824
|
||||
51510 Alexandria city VA 160662
|
||||
51540 Charlottesville city VA 44388
|
||||
51550 Chesapeake city VA 255332
|
||||
51600 Fairfax city VA 26772
|
||||
51610 Falls Church city VA 15159
|
||||
51650 Hampton city VA 137315
|
||||
51670 Hopewell city VA 23261
|
||||
51680 Lynchburg city VA 81347
|
||||
51700 Newport News city VA 183230
|
||||
51710 Norfolk city VA 231013
|
||||
51730 Petersburg city VA 33734
|
||||
51740 Portsmouth city VA 96777
|
||||
51760 Richmond city VA 237257
|
||||
51800 Suffolk city VA 104699
|
||||
51810 Virginia Beach city VA 453737
|
||||
51840 Winchester city VA 28272
|
||||
53005 Benton County WA 221722
|
||||
53011 Clark County WA 532119
|
||||
53015 Cowlitz County WA 114885
|
||||
53021 Franklin County WA 102612
|
||||
53025 Grant County WA 105727
|
||||
53033 King County WA 2344939
|
||||
53035 Kitsap County WA 283374
|
||||
53041 Lewis County WA 88062
|
||||
53053 Pierce County WA 946288
|
||||
53061 Snohomish County WA 870656
|
||||
53063 Spokane County WA 558344
|
||||
53065 Stevens County WA 49668
|
||||
53067 Thurston County WA 304261
|
||||
53071 Walla Walla County WA 62361
|
||||
53073 Whatcom County WA 236392
|
||||
53077 Yakima County WA 259185
|
||||
54011 Cabell County WV 91183
|
||||
54039 Kanawha County WV 172381
|
||||
54045 Logan County WV 30375
|
||||
54047 McDowell County WV 16878
|
||||
54059 Mingo County WV 21455
|
||||
54061 Monongalia County WV 107991
|
||||
54069 Ohio County WV 40496
|
||||
54083 Randolph County WV 26882
|
||||
54107 Wood County WV 82385
|
||||
55009 Brown County WI 275803
|
||||
55013 Burnett County WI 17223
|
||||
55017 Chippewa County WI 67614
|
||||
55025 Dane County WI 590375
|
||||
55027 Dodge County WI 89097
|
||||
55035 Eau Claire County WI 109033
|
||||
55049 Iowa County WI 23951
|
||||
55055 Jefferson County WI 86505
|
||||
55059 Kenosha County WI 168448
|
||||
55073 Marathon County WI 139432
|
||||
55079 Milwaukee County WI 924216
|
||||
55087 Outagamie County WI 195894
|
||||
55089 Ozaukee County WI 94346
|
||||
55093 Pierce County WI 43519
|
||||
55095 Polk County WI 45803
|
||||
55101 Racine County WI 198919
|
||||
55105 Rock County WI 166472
|
||||
55115 Shawano County WI 41560
|
||||
55121 Trempealeau County WI 31171
|
||||
55127 Walworth County WI 106127
|
||||
55131 Washington County WI 139238
|
||||
55133 Waukesha County WI 417210
|
||||
55139 Winnebago County WI 174218
|
||||
56001 Albany County WY 38558
|
||||
56005 Campbell County WY 48145
|
||||
56007 Carbon County WY 14013
|
||||
56021 Laramie County WY 102938
|
||||
56025 Natrona County WY 80526
|
||||
56029 Park County WY 31171
|
||||
56037 Sweetwater County WY 41267
|
||||
56039 Teton County WY 23333
|
||||
56041 Uinta County WY 20728
|
||||
|
@@ -1 +0,0 @@
|
||||
{ "format": "{code}", "rows": "postal-code.tsv", "key": "code", "parent": "locality", "weight": "addresses" }
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1 +0,0 @@
|
||||
{ "format": "{name}", "rows": "region.tsv", "key": "abbr", "name": "name", "weight": "population" }
|
||||
@@ -1,51 +0,0 @@
|
||||
abbr code name population timezone
|
||||
AK 02 Alaska 737270 America/Anchorage
|
||||
AL 01 Alabama 5193088 America/Chicago
|
||||
AR 05 Arkansas 3114791 America/Chicago
|
||||
AZ 04 Arizona 7623818 America/Phoenix
|
||||
CA 06 California 39355309 America/Los_Angeles
|
||||
CO 08 Colorado 6012561 America/Denver
|
||||
CT 09 Connecticut 3688496 America/New_York
|
||||
DC 11 District of Columbia 693645 America/New_York
|
||||
DE 10 Delaware 1059952 America/New_York
|
||||
FL 12 Florida 23462518 America/New_York
|
||||
GA 13 Georgia 11302748 America/New_York
|
||||
IA 19 Iowa 3238387 America/Chicago
|
||||
ID 16 Idaho 2029733 America/Boise
|
||||
IL 17 Illinois 12719141 America/Chicago
|
||||
IN 18 Indiana 6973333 America/Indiana/Indianapolis
|
||||
KS 20 Kansas 2977220 America/Chicago
|
||||
KY 21 Kentucky 4606864 America/New_York
|
||||
LA 22 Louisiana 4618189 America/Chicago
|
||||
MA 25 Massachusetts 7154084 America/New_York
|
||||
MD 24 Maryland 6265347 America/New_York
|
||||
ME 23 Maine 1414874 America/New_York
|
||||
MI 26 Michigan 10127884 America/Detroit
|
||||
MN 27 Minnesota 5830405 America/Chicago
|
||||
MO 29 Missouri 6270541 America/Chicago
|
||||
MS 28 Mississippi 2954160 America/Chicago
|
||||
MT 30 Montana 1144694 America/Denver
|
||||
NC 37 North Carolina 11197968 America/New_York
|
||||
ND 38 North Dakota 799358 America/Chicago
|
||||
NE 31 Nebraska 2018006 America/Chicago
|
||||
NH 33 New Hampshire 1415342 America/New_York
|
||||
NJ 34 New Jersey 9548215 America/New_York
|
||||
NM 35 New Mexico 2125498 America/Denver
|
||||
NV 32 Nevada 3282188 America/Los_Angeles
|
||||
NY 36 New York 20002427 America/New_York
|
||||
OH 39 Ohio 11900510 America/New_York
|
||||
OK 40 Oklahoma 4123288 America/Chicago
|
||||
OR 41 Oregon 4273586 America/Los_Angeles
|
||||
PA 42 Pennsylvania 13059432 America/New_York
|
||||
RI 44 Rhode Island 1114521 America/New_York
|
||||
SC 45 South Carolina 5570274 America/New_York
|
||||
SD 46 South Dakota 935094 America/Chicago
|
||||
TN 47 Tennessee 7315076 America/Chicago
|
||||
TX 48 Texas 31709821 America/Chicago
|
||||
UT 49 Utah 3538904 America/Denver
|
||||
VA 51 Virginia 8880107 America/New_York
|
||||
VT 50 Vermont 644663 America/New_York
|
||||
WA 53 Washington 8001020 America/Los_Angeles
|
||||
WI 55 Wisconsin 5972787 America/Chicago
|
||||
WV 54 West Virginia 1766147 America/New_York
|
||||
WY 56 Wyoming 588753 America/Denver
|
||||
|
@@ -1 +0,0 @@
|
||||
{ "format": "{name}", "rows": "street.tsv", "parent": "locality", "weight": "addresses" }
|
||||
-475272
File diff suppressed because it is too large
Load Diff
@@ -1,25 +0,0 @@
|
||||
{
|
||||
"index": {
|
||||
"misc.car": {"paths":["","make","model"]},
|
||||
"misc.coordinate": {"paths":["","lat","lon"]},
|
||||
"misc.creditcard": {"paths":["","d"]},
|
||||
"misc.currency": {"paths":["","code","decimals","name","numeric","symbol"]},
|
||||
"misc.datetime": {"paths":[""]},
|
||||
"misc.emoji": {"paths":[""]},
|
||||
"misc.httpmethod": {"paths":["","idempotent","method","safe"]},
|
||||
"misc.httpstatus": {"paths":["","code","reason"]},
|
||||
"misc.language": {"paths":["","code","code3","name"]},
|
||||
"misc.loglevel": {"paths":["","code","keyword","severity"]},
|
||||
"misc.mac": {"paths":[""]},
|
||||
"misc.mimetype": {"paths":["","ext","type"]},
|
||||
"misc.objectid": {"paths":[""]},
|
||||
"misc.port": {"paths":["","number","service"]},
|
||||
"misc.protocol": {"paths":["","keyword","name","number"]},
|
||||
"misc.territory": {"paths":["","alpha2","alpha3","calling-code","capital","country","currency","flag","languages","name","numeric","timezone","timezone.offset","timezone.territory","timezone.weight","timezone.zone","tld"]},
|
||||
"misc.timezone": {"parent":"territory","paths":["","offset","territory","weight","zone"]},
|
||||
"misc.tld": {"paths":["","tld","type","unicode"]},
|
||||
"misc.useragent": {"paths":["","browser","device","os","ua"]},
|
||||
"misc.uuid": {"paths":["","variant"]}
|
||||
},
|
||||
"namespace": "misc"
|
||||
}
|
||||
+10
-1
@@ -1 +1,10 @@
|
||||
{ "format": "{make} {model}", "rows": "car.tsv" }
|
||||
[
|
||||
{ "format": "{maker} {model}", "maker": "BMW", "model": ["3 Series", "5 Series", "X3", "X5"] },
|
||||
{ "format": "{maker} {model}", "maker": "Ford", "model": ["Fiesta", "Focus", "Mustang", "Explorer"] },
|
||||
{ "format": "{maker} {model}", "maker": "Honda", "model": ["Civic", "Accord", "CR-V", "Jazz"] },
|
||||
{ "format": "{maker} {model}", "maker": "Mercedes-Benz", "model": ["A-Class", "C-Class", "E-Class", "GLC"] },
|
||||
{ "format": "{maker} {model}", "maker": "Tesla", "model": ["Model 3", "Model S", "Model X", "Model Y"] },
|
||||
{ "format": "{maker} {model}", "maker": "Toyota", "model": ["Corolla", "Camry", "RAV4", "Yaris"] },
|
||||
{ "format": "{maker} {model}", "maker": "Volkswagen", "model": ["Golf", "Passat", "Polo", "Tiguan"] },
|
||||
{ "format": "{maker} {model}", "maker": "Volvo", "model": ["XC40", "XC60", "XC90", "V60"] }
|
||||
]
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
make model
|
||||
BMW 3 Series
|
||||
BMW 5 Series
|
||||
BMW X3
|
||||
BMW X5
|
||||
Ford Explorer
|
||||
Ford Fiesta
|
||||
Ford Focus
|
||||
Ford Mustang
|
||||
Honda Accord
|
||||
Honda CR-V
|
||||
Honda Civic
|
||||
Honda Jazz
|
||||
Mercedes-Benz A-Class
|
||||
Mercedes-Benz C-Class
|
||||
Mercedes-Benz E-Class
|
||||
Mercedes-Benz GLC
|
||||
Tesla Model 3
|
||||
Tesla Model S
|
||||
Tesla Model X
|
||||
Tesla Model Y
|
||||
Toyota Camry
|
||||
Toyota Corolla
|
||||
Toyota RAV4
|
||||
Toyota Yaris
|
||||
Volkswagen Golf
|
||||
Volkswagen Passat
|
||||
Volkswagen Polo
|
||||
Volkswagen Tiguan
|
||||
Volvo V60
|
||||
Volvo XC40
|
||||
Volvo XC60
|
||||
Volvo XC90
|
||||
|
@@ -0,0 +1,22 @@
|
||||
[
|
||||
{ "format": "{name}", "name": "Australia", "alpha2": "AU", "alpha3": "AUS" },
|
||||
{ "format": "{name}", "name": "Brazil", "alpha2": "BR", "alpha3": "BRA" },
|
||||
{ "format": "{name}", "name": "Canada", "alpha2": "CA", "alpha3": "CAN" },
|
||||
{ "format": "{name}", "name": "China", "alpha2": "CN", "alpha3": "CHN" },
|
||||
{ "format": "{name}", "name": "Denmark", "alpha2": "DK", "alpha3": "DNK" },
|
||||
{ "format": "{name}", "name": "Finland", "alpha2": "FI", "alpha3": "FIN" },
|
||||
{ "format": "{name}", "name": "France", "alpha2": "FR", "alpha3": "FRA" },
|
||||
{ "format": "{name}", "name": "Germany", "alpha2": "DE", "alpha3": "DEU" },
|
||||
{ "format": "{name}", "name": "India", "alpha2": "IN", "alpha3": "IND" },
|
||||
{ "format": "{name}", "name": "Italy", "alpha2": "IT", "alpha3": "ITA" },
|
||||
{ "format": "{name}", "name": "Japan", "alpha2": "JP", "alpha3": "JPN" },
|
||||
{ "format": "{name}", "name": "Mexico", "alpha2": "MX", "alpha3": "MEX" },
|
||||
{ "format": "{name}", "name": "Netherlands", "alpha2": "NL", "alpha3": "NLD" },
|
||||
{ "format": "{name}", "name": "Norway", "alpha2": "NO", "alpha3": "NOR" },
|
||||
{ "format": "{name}", "name": "Poland", "alpha2": "PL", "alpha3": "POL" },
|
||||
{ "format": "{name}", "name": "Spain", "alpha2": "ES", "alpha3": "ESP" },
|
||||
{ "format": "{name}", "name": "Sweden", "alpha2": "SE", "alpha3": "SWE" },
|
||||
{ "format": "{name}", "name": "Switzerland", "alpha2": "CH", "alpha3": "CHE" },
|
||||
{ "format": "{name}", "name": "United Kingdom", "alpha2": "GB", "alpha3": "GBR" },
|
||||
{ "format": "{name}", "name": "United States", "alpha2": "US", "alpha3": "USA" }
|
||||
]
|
||||
+18
-1
@@ -1 +1,18 @@
|
||||
{ "format": "{code}", "rows": "currency.tsv", "key": "code", "name": "name" }
|
||||
[
|
||||
{ "format": "{code}", "code": "AUD", "name": "Australian Dollar", "symbol": "$" },
|
||||
{ "format": "{code}", "code": "BRL", "name": "Brazilian Real", "symbol": "R$" },
|
||||
{ "format": "{code}", "code": "CAD", "name": "Canadian Dollar", "symbol": "$" },
|
||||
{ "format": "{code}", "code": "CHF", "name": "Swiss Franc", "symbol": "CHF" },
|
||||
{ "format": "{code}", "code": "CNY", "name": "Chinese Yuan", "symbol": "¥" },
|
||||
{ "format": "{code}", "code": "DKK", "name": "Danish Krone", "symbol": "kr" },
|
||||
{ "format": "{code}", "code": "EUR", "name": "Euro", "symbol": "€" },
|
||||
{ "format": "{code}", "code": "GBP", "name": "Pound Sterling", "symbol": "£" },
|
||||
{ "format": "{code}", "code": "INR", "name": "Indian Rupee", "symbol": "₹" },
|
||||
{ "format": "{code}", "code": "JPY", "name": "Japanese Yen", "symbol": "¥" },
|
||||
{ "format": "{code}", "code": "MXN", "name": "Mexican Peso", "symbol": "$" },
|
||||
{ "format": "{code}", "code": "NOK", "name": "Norwegian Krone", "symbol": "kr" },
|
||||
{ "format": "{code}", "code": "PLN", "name": "Polish Zloty", "symbol": "zł" },
|
||||
{ "format": "{code}", "code": "SEK", "name": "Swedish Krona", "symbol": "kr" },
|
||||
{ "format": "{code}", "code": "USD", "name": "US Dollar", "symbol": "$" },
|
||||
{ "format": "{code}", "code": "ZAR", "name": "South African Rand", "symbol": "R" }
|
||||
]
|
||||
|
||||
@@ -1,166 +0,0 @@
|
||||
code decimals name numeric symbol
|
||||
AED 2 UAE Dirham 784 AED
|
||||
AFN 2 Afghani 971 ؋
|
||||
ALL 2 Lek 008 ALL
|
||||
AMD 2 Armenian Dram 051 ֏
|
||||
AOA 2 Kwanza 973 Kz
|
||||
ARS 2 Argentine Peso 032 $
|
||||
AUD 2 Australian Dollar 036 $
|
||||
AWG 2 Aruban Florin 533 AWG
|
||||
AZN 2 Azerbaijan Manat 944 ₼
|
||||
BAM 2 Convertible Mark 977 KM
|
||||
BBD 2 Barbados Dollar 052 $
|
||||
BDT 2 Taka 050 ৳
|
||||
BHD 3 Bahraini Dinar 048 BHD
|
||||
BIF 0 Burundi Franc 108 BIF
|
||||
BMD 2 Bermudian Dollar 060 $
|
||||
BND 2 Brunei Dollar 096 $
|
||||
BOB 2 Boliviano 068 Bs
|
||||
BOV 2 Mvdol 984 BOV
|
||||
BRL 2 Brazilian Real 986 R$
|
||||
BSD 2 Bahamian Dollar 044 $
|
||||
BTN 2 Ngultrum 064 BTN
|
||||
BWP 2 Pula 072 P
|
||||
BYN 2 Belarusian Ruble 933 BYN
|
||||
BZD 2 Belize Dollar 084 $
|
||||
CAD 2 Canadian Dollar 124 $
|
||||
CDF 2 Congolese Franc 976 CDF
|
||||
CHE 2 WIR Euro 947 CHE
|
||||
CHF 2 Swiss Franc 756 Fr.
|
||||
CHW 2 WIR Franc 948 CHW
|
||||
CLF 4 Unidad de Fomento 990 CLF
|
||||
CLP 0 Chilean Peso 152 $
|
||||
CNY 2 Yuan Renminbi 156 ¥
|
||||
COP 2 Colombian Peso 170 $
|
||||
COU 2 Unidad de Valor Real 970 COU
|
||||
CRC 2 Costa Rican Colon 188 ₡
|
||||
CUP 2 Cuban Peso 192 $
|
||||
CVE 2 Cabo Verde Escudo 132 CVE
|
||||
CZK 2 Czech Koruna 203 Kč
|
||||
DJF 0 Djibouti Franc 262 DJF
|
||||
DKK 2 Danish Krone 208 kr
|
||||
DOP 2 Dominican Peso 214 $
|
||||
DZD 2 Algerian Dinar 012 DZD
|
||||
EGP 2 Egyptian Pound 818 E£
|
||||
ERN 2 Nakfa 232 ERN
|
||||
ETB 2 Ethiopian Birr 230 ETB
|
||||
EUR 2 Euro 978 €
|
||||
FJD 2 Fiji Dollar 242 $
|
||||
FKP 2 Falkland Islands Pound 238 £
|
||||
GBP 2 Pound Sterling 826 £
|
||||
GEL 2 Lari 981 ₾
|
||||
GHS 2 Ghana Cedi 936 GH₵
|
||||
GIP 2 Gibraltar Pound 292 £
|
||||
GMD 2 Dalasi 270 GMD
|
||||
GNF 0 Guinean Franc 324 FG
|
||||
GTQ 2 Quetzal 320 Q
|
||||
GYD 2 Guyana Dollar 328 $
|
||||
HKD 2 Hong Kong Dollar 344 $
|
||||
HNL 2 Lempira 340 L
|
||||
HTG 2 Gourde 332 HTG
|
||||
HUF 2 Forint 348 Ft
|
||||
IDR 2 Rupiah 360 Rp
|
||||
ILS 2 New Israeli Sheqel 376 ₪
|
||||
INR 2 Indian Rupee 356 ₹
|
||||
IQD 3 Iraqi Dinar 368 IQD
|
||||
IRR 2 Iranian Rial 364 IRR
|
||||
ISK 0 Iceland Krona 352 kr
|
||||
JMD 2 Jamaican Dollar 388 $
|
||||
JOD 3 Jordanian Dinar 400 JOD
|
||||
JPY 0 Yen 392 ¥
|
||||
KES 2 Kenyan Shilling 404 KES
|
||||
KGS 2 Som 417 ⃀
|
||||
KHR 2 Riel 116 ៛
|
||||
KMF 0 Comorian Franc 174 CF
|
||||
KPW 2 North Korean Won 408 ₩
|
||||
KRW 0 Won 410 ₩
|
||||
KWD 3 Kuwaiti Dinar 414 KWD
|
||||
KYD 2 Cayman Islands Dollar 136 $
|
||||
KZT 2 Tenge 398 ₸
|
||||
LAK 2 Lao Kip 418 ₭
|
||||
LBP 2 Lebanese Pound 422 L£
|
||||
LKR 2 Sri Lanka Rupee 144 Rs
|
||||
LRD 2 Liberian Dollar 430 $
|
||||
LSL 2 Loti 426 LSL
|
||||
LYD 3 Libyan Dinar 434 LYD
|
||||
MAD 2 Moroccan Dirham 504 MAD
|
||||
MDL 2 Moldovan Leu 498 MDL
|
||||
MGA 2 Malagasy Ariary 969 Ar
|
||||
MKD 2 Denar 807 MKD
|
||||
MMK 2 Kyat 104 K
|
||||
MNT 2 Tugrik 496 ₮
|
||||
MOP 2 Pataca 446 MOP
|
||||
MRU 2 Ouguiya 929 MRU
|
||||
MUR 2 Mauritius Rupee 480 Rs
|
||||
MVR 2 Rufiyaa 462 MVR
|
||||
MWK 2 Malawi Kwacha 454 MWK
|
||||
MXN 2 Mexican Peso 484 $
|
||||
MXV 2 Mexican Unidad de Inversion (UDI) 979 MXV
|
||||
MYR 2 Malaysian Ringgit 458 RM
|
||||
MZN 2 Mozambique Metical 943 MZN
|
||||
NAD 2 Namibia Dollar 516 $
|
||||
NGN 2 Naira 566 ₦
|
||||
NIO 2 Cordoba Oro 558 C$
|
||||
NOK 2 Norwegian Krone 578 kr
|
||||
NPR 2 Nepalese Rupee 524 Rs
|
||||
NZD 2 New Zealand Dollar 554 $
|
||||
OMR 3 Rial Omani 512 OMR
|
||||
PAB 2 Balboa 590 PAB
|
||||
PEN 2 Sol 604 PEN
|
||||
PGK 2 Kina 598 PGK
|
||||
PHP 2 Philippine Peso 608 ₱
|
||||
PKR 2 Pakistan Rupee 586 Rs
|
||||
PLN 2 Zloty 985 zł
|
||||
PYG 0 Guarani 600 ₲
|
||||
QAR 2 Qatari Rial 634 QAR
|
||||
RON 2 Romanian Leu 946 lei
|
||||
RSD 2 Serbian Dinar 941 RSD
|
||||
RUB 2 Russian Ruble 643 ₽
|
||||
RWF 0 Rwanda Franc 646 RF
|
||||
SAR 2 Saudi Riyal 682 SAR
|
||||
SBD 2 Solomon Islands Dollar 090 $
|
||||
SCR 2 Seychelles Rupee 690 SCR
|
||||
SDG 2 Sudanese Pound 938 SDG
|
||||
SEK 2 Swedish Krona 752 kr
|
||||
SGD 2 Singapore Dollar 702 $
|
||||
SHP 2 Saint Helena Pound 654 £
|
||||
SLE 2 Leone 925 SLE
|
||||
SOS 2 Somali Shilling 706 SOS
|
||||
SRD 2 Surinam Dollar 968 $
|
||||
SSP 2 South Sudanese Pound 728 £
|
||||
STN 2 Dobra 930 Db
|
||||
SVC 2 El Salvador Colon 222 SVC
|
||||
SYP 2 Syrian Pound 760 £
|
||||
SZL 2 Lilangeni 748 SZL
|
||||
THB 2 Baht 764 ฿
|
||||
TJS 2 Somoni 972 TJS
|
||||
TMT 2 Turkmenistan New Manat 934 TMT
|
||||
TND 3 Tunisian Dinar 788 TND
|
||||
TOP 2 Pa’anga 776 T$
|
||||
TRY 2 Turkish Lira 949 ₺
|
||||
TTD 2 Trinidad and Tobago Dollar 780 $
|
||||
TWD 2 New Taiwan Dollar 901 $
|
||||
TZS 2 Tanzanian Shilling 834 TZS
|
||||
UAH 2 Hryvnia 980 ₴
|
||||
UGX 0 Uganda Shilling 800 UGX
|
||||
USD 2 US Dollar 840 $
|
||||
USN 2 US Dollar (Next day) 997 USN
|
||||
UYI 0 Uruguay Peso en Unidades Indexadas (UI) 940 UYI
|
||||
UYU 2 Peso Uruguayo 858 $
|
||||
UYW 4 Unidad Previsional 927 UYW
|
||||
UZS 2 Uzbekistan Sum 860 UZS
|
||||
VED 2 Bolívar Soberano 926 VED
|
||||
VES 2 Bolívar Soberano 928 VES
|
||||
VND 0 Dong 704 ₫
|
||||
VUV 0 Vatu 548 VUV
|
||||
WST 2 Tala 882 WST
|
||||
XAD 2 Arab Accounting Dinar 396 XAD
|
||||
XAF 0 CFA Franc BEAC 950 FCFA
|
||||
XCD 2 East Caribbean Dollar 951 $
|
||||
XCG 2 Caribbean Guilder 532 Cg.
|
||||
XOF 0 CFA Franc BCEAO 952 F CFA
|
||||
XPF 0 CFP Franc 953 CFPF
|
||||
YER 2 Yemeni Rial 886 YER
|
||||
ZAR 2 Rand 710 R
|
||||
ZMW 2 Zambian Kwacha 967 ZK
|
||||
ZWG 2 Zimbabwe Gold 924 ZWG
|
||||
|
@@ -1 +0,0 @@
|
||||
"{date(2000-01-01,2029-12-31,'2006-01-02T15:04:05Z')}"
|
||||
@@ -1,15 +0,0 @@
|
||||
// Package misc is fejkdata's locale-neutral module, whose categories are misc.*.
|
||||
package misc
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"io/fs"
|
||||
)
|
||||
|
||||
// FS holds the module's categories and its manifest.
|
||||
//
|
||||
//go:embed *.json *.tsv .fejkdata.json
|
||||
var FS embed.FS
|
||||
|
||||
// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
|
||||
func Modules() []fs.FS { return []fs.FS{FS} }
|
||||
@@ -1,22 +0,0 @@
|
||||
package misc_test
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"testing"
|
||||
|
||||
"github.com/larvit/fejkdata"
|
||||
"github.com/larvit/fejkdata/data/misc"
|
||||
)
|
||||
|
||||
func TestFSLoadsByItsManifest(t *testing.T) {
|
||||
if _, err := fs.Stat(misc.FS, ".fejkdata.json"); err != nil {
|
||||
t.Fatalf("FS holds no manifest: %v", err)
|
||||
}
|
||||
f, err := fejkdata.New(fejkdata.WithDataFS(misc.Modules()...), fejkdata.WithSeed(1))
|
||||
if err != nil {
|
||||
t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
|
||||
}
|
||||
if len(f.List()) == 0 {
|
||||
t.Error("List() is empty, want the module's categories")
|
||||
}
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
{ "format": "{method}", "rows": "httpmethod.tsv", "key": "method" }
|
||||
@@ -1,10 +0,0 @@
|
||||
idempotent method safe
|
||||
false CONNECT false
|
||||
true DELETE false
|
||||
true GET true
|
||||
true HEAD true
|
||||
true OPTIONS true
|
||||
false PATCH false
|
||||
false POST false
|
||||
true PUT false
|
||||
true TRACE true
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user