Make great documentation goal 1.2, and plan the README's reorganisation and the nested-template gaps #33
@@ -0,0 +1,162 @@
|
|||||||
|
name: Packages
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
tag:
|
||||||
|
type: string
|
||||||
|
required: true
|
||||||
|
secrets:
|
||||||
|
TAP_GITHUB_TOKEN:
|
||||||
|
required: true
|
||||||
|
PACKAGES_GPG_KEY:
|
||||||
|
required: true
|
||||||
|
PACKAGES_RSA_KEY:
|
||||||
|
required: true
|
||||||
|
DOCKERHUB_TOKEN:
|
||||||
|
required: true
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
tag:
|
||||||
|
description: The released tag to package, such as v0.1.0
|
||||||
|
type: string
|
||||||
|
required: true
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: packages
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
packages:
|
||||||
|
name: Binaries, packages and image for ${{ inputs.tag }}
|
||||||
|
runs-on: ubuntu-26.04
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
packages: write
|
||||||
|
outputs:
|
||||||
|
latest: ${{ steps.latest.outputs.latest }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v7
|
||||||
|
with:
|
||||||
|
ref: ${{ inputs.tag }}
|
||||||
|
path: src
|
||||||
|
fetch-depth: 0
|
||||||
|
persist-credentials: false
|
||||||
|
# Check out the packaging config at the workflow's commit, so a tag cut before the
|
||||||
|
# config existed can still be packaged.
|
||||||
|
- uses: actions/checkout@v7
|
||||||
|
with:
|
||||||
|
path: tooling
|
||||||
|
persist-credentials: false
|
||||||
|
- name: The Go version the gate tests
|
||||||
|
id: go
|
||||||
|
run: sed -n 's/^ARG GO_VERSION=/version=/p' tooling/Dockerfile >> "$GITHUB_OUTPUT"
|
||||||
|
- uses: actions/setup-go@v7
|
||||||
|
with:
|
||||||
|
go-version: ${{ steps.go.outputs.version }}
|
||||||
|
cache: false
|
||||||
|
- name: Channels holding one version
|
||||||
|
id: latest
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ github.token }}
|
||||||
|
TAG: ${{ inputs.tag }}
|
||||||
|
run: |
|
||||||
|
latest="$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName --jq .tagName)"
|
||||||
|
if [ "$latest" = "$TAG" ]; then
|
||||||
|
echo "latest=true" >> "$GITHUB_OUTPUT"
|
||||||
|
else
|
||||||
|
echo "skip=--skip=homebrew,scoop" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "OLDER_TAG=1" >> "$GITHUB_ENV"
|
||||||
|
fi
|
||||||
|
- uses: docker/setup-buildx-action@v4
|
||||||
|
- uses: docker/login-action@v4
|
||||||
|
with:
|
||||||
|
registry: ghcr.io
|
||||||
|
username: ${{ github.actor }}
|
||||||
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
- uses: goreleaser/goreleaser-action@v7
|
||||||
|
with:
|
||||||
|
version: v2.18.3
|
||||||
|
workdir: src
|
||||||
|
args: release --clean --config ../tooling/release-tooling/goreleaser.yaml ${{ steps.latest.outputs.skip }}
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }}
|
||||||
|
- name: The built binary and the published image print the tag
|
||||||
|
env:
|
||||||
|
TAG: ${{ inputs.tag }}
|
||||||
|
run: |
|
||||||
|
test "$(dist/fejkdata_linux_amd64_v1/fejkdata --version)" = "fejkdata $TAG"
|
||||||
|
test "$(docker run --rm "ghcr.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG"
|
||||||
|
|
||||||
|
dockerhub:
|
||||||
|
name: Docker Hub image for ${{ inputs.tag }}
|
||||||
|
needs: packages
|
||||||
|
runs-on: ubuntu-26.04
|
||||||
|
permissions:
|
||||||
|
packages: read
|
||||||
|
steps:
|
||||||
|
- uses: docker/login-action@v4
|
||||||
|
with:
|
||||||
|
registry: ghcr.io
|
||||||
|
username: ${{ github.actor }}
|
||||||
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
- uses: docker/login-action@v4
|
||||||
|
with:
|
||||||
|
username: ${{ vars.DOCKERHUB_USERNAME }}
|
||||||
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||||
|
- name: Copy the GHCR image to Docker Hub, and check it prints the tag
|
||||||
|
env:
|
||||||
|
LATEST: ${{ needs.packages.outputs.latest }}
|
||||||
|
TAG: ${{ inputs.tag }}
|
||||||
|
run: |
|
||||||
|
tags=(-t "docker.io/larvit/fejkdata:${TAG#v}")
|
||||||
|
if [ "$LATEST" = true ]; then
|
||||||
|
tags+=(-t docker.io/larvit/fejkdata:latest)
|
||||||
|
fi
|
||||||
|
docker buildx imagetools create "${tags[@]}" "ghcr.io/larvit/fejkdata:${TAG#v}"
|
||||||
|
test "$(docker run --rm "docker.io/larvit/fejkdata:${TAG#v}" --version)" = "fejkdata $TAG"
|
||||||
|
|
||||||
|
repos:
|
||||||
|
name: apt, dnf and apk repositories for ${{ inputs.tag }}
|
||||||
|
needs: packages
|
||||||
|
if: needs.packages.outputs.latest == 'true'
|
||||||
|
runs-on: ubuntu-26.04
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
pages: write
|
||||||
|
id-token: write
|
||||||
|
environment:
|
||||||
|
name: github-pages
|
||||||
|
url: ${{ steps.deploy.outputs.page_url }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v7
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
- name: The release's Linux packages
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ github.token }}
|
||||||
|
TAG: ${{ inputs.tag }}
|
||||||
|
run: gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --dir packages --pattern '*.deb' --pattern '*.rpm' --pattern '*.apk'
|
||||||
|
- uses: docker/setup-buildx-action@v4
|
||||||
|
- name: Install fejkdata from each repository, then export them
|
||||||
|
env:
|
||||||
|
PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }}
|
||||||
|
PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }}
|
||||||
|
TAG: ${{ inputs.tag }}
|
||||||
|
run: |
|
||||||
|
build() {
|
||||||
|
docker buildx build -f release-tooling/package-repos.Dockerfile \
|
||||||
|
--secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY \
|
||||||
|
--build-context packages=packages --build-arg "EXPECTED_VERSION=$TAG" "$@" .
|
||||||
|
}
|
||||||
|
build --target checks --output type=local,dest=checks
|
||||||
|
build --target site --output type=local,dest=site
|
||||||
|
- uses: actions/upload-pages-artifact@v5
|
||||||
|
with:
|
||||||
|
path: site
|
||||||
|
- id: deploy
|
||||||
|
uses: actions/deploy-pages@v5
|
||||||
+85
-14
@@ -4,6 +4,9 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
push:
|
push:
|
||||||
branches: [main]
|
branches: [main]
|
||||||
|
workflow_dispatch:
|
||||||
|
schedule:
|
||||||
|
- cron: '17 5 * * *'
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
@@ -11,9 +14,10 @@ permissions:
|
|||||||
jobs:
|
jobs:
|
||||||
test:
|
test:
|
||||||
name: vet + fmt + tests
|
name: vet + fmt + tests
|
||||||
runs-on: ubuntu-24.04
|
if: github.event_name != 'schedule'
|
||||||
|
runs-on: ubuntu-26.04
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
@@ -26,11 +30,16 @@ jobs:
|
|||||||
else
|
else
|
||||||
base='${{ github.event.before }}'
|
base='${{ github.event.before }}'
|
||||||
fi
|
fi
|
||||||
if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum Dockerfile .dockerignore .github data testdata README.md; then
|
if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum Dockerfile .dockerignore .github data release-tooling testdata vulnerability-filing README.md todo.md; then
|
||||||
echo "code=false" >> "$GITHUB_OUTPUT"
|
echo "code=false" >> "$GITHUB_OUTPUT"
|
||||||
else
|
else
|
||||||
echo "code=true" >> "$GITHUB_OUTPUT"
|
echo "code=true" >> "$GITHUB_OUTPUT"
|
||||||
fi
|
fi
|
||||||
|
if git diff --quiet "$base"...HEAD -- '*.go' go.mod go.sum release-tooling Dockerfile .dockerignore LICENSE DATA-LICENSES.md .github/workflows/packages.yml .github/workflows/test.yml; then
|
||||||
|
echo "packaging=false" >> "$GITHUB_OUTPUT"
|
||||||
|
else
|
||||||
|
echo "packaging=true" >> "$GITHUB_OUTPUT"
|
||||||
|
fi
|
||||||
echo "base=$base" >> "$GITHUB_OUTPUT"
|
echo "base=$base" >> "$GITHUB_OUTPUT"
|
||||||
- name: CHANGELOG entry for data changes
|
- name: CHANGELOG entry for data changes
|
||||||
run: |
|
run: |
|
||||||
@@ -48,22 +57,84 @@ jobs:
|
|||||||
# failure is version-specific.
|
# failure is version-specific.
|
||||||
- name: Lowest supported Go
|
- name: Lowest supported Go
|
||||||
if: ${{ !cancelled() && steps.changes.outputs.code == 'true' }}
|
if: ${{ !cancelled() && steps.changes.outputs.code == 'true' }}
|
||||||
run: docker build --build-arg GO_VERSION=1.22.12 .
|
run: docker build --build-arg GO_VERSION=1.22.12 --target portable .
|
||||||
|
- name: Package repositories, signed with throwaway keys
|
||||||
|
if: ${{ !cancelled() && steps.changes.outputs.packaging == 'true' }}
|
||||||
|
run: |
|
||||||
|
docker build -f release-tooling/package-repos.Dockerfile --target throwaway-keys --output "$RUNNER_TEMP/keys" .
|
||||||
|
PACKAGES_GPG_KEY="$(cat "$RUNNER_TEMP/keys/gpg.asc")" PACKAGES_RSA_KEY="$(cat "$RUNNER_TEMP/keys/rsa.pem")" \
|
||||||
|
docker build -f release-tooling/package-repos.Dockerfile --target checks \
|
||||||
|
--secret id=gpg,env=PACKAGES_GPG_KEY --secret id=rsa,env=PACKAGES_RSA_KEY .
|
||||||
|
|
||||||
release:
|
vulnerabilities:
|
||||||
name: Gitea release from CHANGELOG.md
|
name: Known vulnerabilities in this Go version and the required modules
|
||||||
needs: test
|
runs-on: ubuntu-26.04
|
||||||
if: github.event_name == 'push'
|
timeout-minutes: 10
|
||||||
runs-on: ubuntu-24.04
|
|
||||||
permissions:
|
permissions:
|
||||||
|
actions: write
|
||||||
contents: write
|
contents: write
|
||||||
|
pull-requests: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
- run: python3 release-tooling/publish_release.py
|
- name: Fetch govulncheck's report from vuln.go.dev
|
||||||
|
run: docker build --no-cache-filter vulnerabilities --target vulnerability-report --output "$RUNNER_TEMP/vulnerabilities" .
|
||||||
|
- name: File each one in todo.md
|
||||||
|
id: file
|
||||||
|
run: python3 vulnerability-filing/file_vulnerabilities.py "$RUNNER_TEMP/vulnerabilities/report.json" todo.md
|
||||||
|
- name: Propose the filed items
|
||||||
|
if: failure() && steps.file.outcome == 'failure' && github.ref == 'refs/heads/main'
|
||||||
env:
|
env:
|
||||||
GITEA_API_URL: ${{ github.api_url }}
|
GH_TOKEN: ${{ github.token }}
|
||||||
GITEA_REPOSITORY: ${{ github.repository }}
|
run: |
|
||||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
if git diff --quiet todo.md || [ "$(gh pr list --head vulnerabilities --state open --json number --jq length)" != 0 ]; then
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
git config user.name 'github-actions[bot]'
|
||||||
|
git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
|
||||||
|
git switch -C vulnerabilities
|
||||||
|
git commit -qam "File the known vulnerabilities govulncheck reports"
|
||||||
|
git push -qf "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" vulnerabilities
|
||||||
|
gh pr create --head vulnerabilities --title "File the known vulnerabilities govulncheck reports" \
|
||||||
|
--body "govulncheck reports known vulnerabilities in the Go version or a required module, and no release is cut until they are gone. This files them atop the next release in todo.md, scored 0.0 until rank_todo.py scores them; each item names the bump that fixes it."
|
||||||
|
# A pull request the workflow's token opens starts no workflow.
|
||||||
|
gh workflow run test.yml --ref vulnerabilities
|
||||||
|
|
||||||
|
release:
|
||||||
|
name: Release from CHANGELOG.md
|
||||||
|
needs: [test, vulnerabilities]
|
||||||
|
if: github.event_name == 'push'
|
||||||
|
runs-on: ubuntu-26.04
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
outputs:
|
||||||
|
tag: ${{ steps.publish.outputs.tag }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v7
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
- id: publish
|
||||||
|
run: python3 release-tooling/publish_release.py
|
||||||
|
env:
|
||||||
|
FORGE_API_URL: ${{ github.api_url }}
|
||||||
|
FORGE_REPOSITORY: ${{ github.repository }}
|
||||||
|
FORGE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
SHA: ${{ github.sha }}
|
SHA: ${{ github.sha }}
|
||||||
|
|
||||||
|
packages:
|
||||||
|
needs: release
|
||||||
|
if: needs.release.outputs.tag != ''
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
packages: write
|
||||||
|
pages: write
|
||||||
|
id-token: write
|
||||||
|
uses: ./.github/workflows/packages.yml
|
||||||
|
with:
|
||||||
|
tag: ${{ needs.release.outputs.tag }}
|
||||||
|
secrets:
|
||||||
|
TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }}
|
||||||
|
PACKAGES_GPG_KEY: ${{ secrets.PACKAGES_GPG_KEY }}
|
||||||
|
PACKAGES_RSA_KEY: ${{ secrets.PACKAGES_RSA_KEY }}
|
||||||
|
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||||
|
|||||||
@@ -1,27 +1,137 @@
|
|||||||
# Rules
|
# Rules
|
||||||
|
|
||||||
- Go runs only through `docker compose run --rm <test|vet|fmt|build|cyclo>`; the merge gate is `docker build .` plus CI's changelog check.
|
- Go runs only through `docker compose run --rm <test|vet|fmt|build|cyclo|generate|vulncheck>`; the merge gate is what the README's Development section lists.
|
||||||
- Tests first, in their own commit; the implementation follows in the next. A re-pin of seeded output or of `testdata/shipped_shape.txt` is its own commit.
|
- Tests first, in their own commit; the implementation follows in the next. A re-pin of seeded output or of `testdata/shipped_shape.txt` is its own commit.
|
||||||
- A change under `data/` or to the shape pin adds its `CHANGELOG.md` entry under `Unreleased` in the same PR, and so does a change to a flag, an exit code, an exported name, a fence, a builtin or the lowest Go; what is major is the README's Versioning table.
|
- A change under `data/` or to the shape pin adds its `CHANGELOG.md` entry under `Unreleased` in the same PR, and so does a change to a flag, an exit code, an exported name, a fence, a builtin or the lowest Go; what is major is the README's Versioning table.
|
||||||
- One-line commit messages: no ticket prefix, no repo name, no authorship trailers.
|
- One-line commit messages: no ticket prefix, no repo name, no authorship trailers.
|
||||||
- Merges are fast-forward only, which `tea pr merge` cannot spell — every style it
|
- GitHub is where the project lives, so `gh` is the tool: merges are fast-forward
|
||||||
offers fails with "is it still open?", as does a merge whose required checks are
|
only, `gh pr merge --rebase` where the branch is already on top of main.
|
||||||
still pending, so read the checks before believing the style is the problem. Use
|
- CodeRabbit skips every pull request until the repository has 10 GitHub stars: check
|
||||||
`tea api /repos/{owner}/{repo}/pulls/<n>/merge -f Do=fast-forward-only`.
|
its status once when a PR is marked ready, answer a review if one is there, and never
|
||||||
- Gitea queues Actions runs that `/repos/{owner}/{repo}/actions/tasks` does not
|
wait for one.
|
||||||
list, so an empty task list says nothing about whether CI ran; read
|
- From 0.2.0, README goal 3 is gated at 7.0 on the `comprehension-panel` skill, and below it
|
||||||
`/repos/{owner}/{repo}/commits/<sha>/status`.
|
nothing else merges — no feature, no category, no data — bar a security fix, a
|
||||||
- README goal 2 is gated at 7.0 on the `comprehension-panel` skill, and below it
|
dependency bump, a repair of behaviour the README documents, the infrastructure
|
||||||
nothing else merges — no feature, no category, no data — bar a security fix and a
|
the round itself runs on, and a chunk the comprehension-floor decision lists as
|
||||||
dependency bump. `todo.md` carries the round: the nine-seat findings run at depth 1,
|
shipped under the floor. `todo.md` carries the round: a PR per item it plans, the nine-seat
|
||||||
a PR per item it names, then the run again, until the score passes. At or above 7.0
|
findings at depth 1 or a restructure the maintainer approved, then the nine-seat run
|
||||||
|
again, until the score passes. At or above 7.0
|
||||||
every pull request scores with the four-seat run and answers it in one run. No merge
|
every pull request scores with the four-seat run and answers it in one run. No merge
|
||||||
lowers the last score.
|
lowers the last score its own panel recorded: the nine-seat round and the four-seat
|
||||||
|
run are two ratchets, never compared with each other. A lower score blocks the merge
|
||||||
|
even where it may be panel noise: fix what the seats name until the score rises. A PR
|
||||||
|
that only records a run's scores and files what its seats name merges even when a
|
||||||
|
score fell. Through 0.1.0, a merge is blocked only by a paired ruling that rules worse.
|
||||||
- Hard tabs. No comment by default; delete a restatement, a rationale, history, or a file preamble.
|
- Hard tabs. No comment by default; delete a restatement, a rationale, history, or a file preamble.
|
||||||
- One spelling per result: reject the other at `New`, and let the error name the spelling to use.
|
|
||||||
- A standing choice a reader would relitigate goes under Decisions in the README, not in a comment.
|
|
||||||
- Prose naming a source states what that source states: read the register's own field
|
- Prose naming a source states what that source states: read the register's own field
|
||||||
before paraphrasing it, and name the register you read before claiming none
|
before paraphrasing it, and name the register you read before claiming none
|
||||||
publishes a spelling.
|
publishes a spelling.
|
||||||
- A README example is a `json` block that loads and renders as a category; `readme_test.go` runs every one.
|
- A README example is a `json` block that loads and renders as a category; `readme_test.go` runs every one.
|
||||||
- Cyclomatic complexity is gated at 14: the table-shaped dispatches (`linkParent`, `compileTemplate`, `renderEdges`) sit at it and stay whole; a function that would pass it is decomposed.
|
- Cyclomatic complexity is gated at 14: split a function that would go over it. A table-shaped dispatch, a switch with one case per kind of its input, may sit at 14 and stays whole.
|
||||||
|
|
||||||
|
# Decisions
|
||||||
|
|
||||||
|
In [docs/decisions.md](docs/decisions.md):
|
||||||
|
|
||||||
|
- GitHub is canonical, and the module path names it
|
||||||
|
- The Gitea copy stays, as a pull mirror of GitHub
|
||||||
|
- The vocabulary sits below `doc.go`'s package clause, not in the package doc
|
||||||
|
- Options and fields share one namespace
|
||||||
|
- `{a|b}` stays beside nested choices
|
||||||
|
- Flags follow getopt_long
|
||||||
|
- A struct tag is a template by its shape
|
||||||
|
- The CLI renders exactly the template on its stdin, and adds nothing
|
||||||
|
- The CLI reads only the library's public API
|
||||||
|
- A template that does not compile is misuse (exit 2), including a reference that resolves to nothing, save a lone reference under the CLI's `--format` until item 129
|
||||||
|
- Whitespace around a JSON object, array, string or null is dropped, and a bare number or boolean is a format string
|
||||||
|
- `FakeTemplate` and `NewTemplate` both stay
|
||||||
|
- The shipped data is embedded, not discovered
|
||||||
|
- The shipped data is Go packages a library imports by choice, and the CLI carries every one, from 0.1.0
|
||||||
|
- A shipped package's folder spells its namespace, its Go name is that namespace in lower case without separators, and its manifest mounts it with `namespace`
|
||||||
|
- A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads
|
||||||
|
- Every `{…}` draws afresh, and only a name keeps a pick
|
||||||
|
- Reference sigils follow the filesystem
|
||||||
|
- A change to what exists is a major; a minor only adds
|
||||||
|
- An error is a contract by what it names, not its bytes
|
||||||
|
- A load reports every mistake at once
|
||||||
|
- Formats are checked after all data is loaded, so their errors name the category
|
||||||
|
- Raising the lowest supported Go is a major
|
||||||
|
- The format check runs on the latest Go only
|
||||||
|
- The changelog heading is the one spelling of a release; CI cuts the tag
|
||||||
|
- A `--data-path` override rebinds every reference to the category it replaces
|
||||||
|
- The repeat cap bounds renders, not bytes
|
||||||
|
- 64-bit targets only
|
||||||
|
- A constant zero divisor is a load error; in a string column a divisor that is not constant prints `Inf`
|
||||||
|
- Samples say what they emit, transforms what they do
|
||||||
|
- A record is a template seen as columns; a Go struct is the one second schema
|
||||||
|
- A struct's records follow Go's field access, and compile on first use
|
||||||
|
- A category may reference itself, and a record's fences run at load
|
||||||
|
- A record's column set is fixed before the first draw
|
||||||
|
- Null is a `null` item, not a rate
|
||||||
|
- A typed column holds one value, not composed text
|
||||||
|
- A column that only reads one reference or name is the column it reads
|
||||||
|
- A typed column's calc is refused unless proven
|
||||||
|
- `Column` carries text, not a Go value
|
||||||
|
- A unit takes the stem of what it is, and a file the stem of the units it holds
|
||||||
|
- One name, one meaning
|
||||||
|
- The performance gate asserts allocations, not wall-clock time
|
||||||
|
- Rows live in a TSV, the shape in JSON
|
||||||
|
- A selector is bracketed, and a dot inside it is literal
|
||||||
|
- `parent` names the link column and the table alike
|
||||||
|
- After a row, a path names a column or a linked table
|
||||||
|
- A selected row is fixed, not drawn
|
||||||
|
- `..` steps up to the parent row, and a step down after it draws afresh, or once per name's pick
|
||||||
|
- A table selects by one key column, a code, never a free-form name
|
||||||
|
- A parent row with no child row is a load error
|
||||||
|
- A table is a record of string columns
|
||||||
|
- The key map is built at load, the rest on first draw
|
||||||
|
- Two categories may name one TSV
|
||||||
|
- A cell may read a reference, and each row its own
|
||||||
|
- A path is walked once without drawing before it is walked for real
|
||||||
|
- A country's postal codes and streets are siblings under its locality
|
||||||
|
- A locale package imports its country's geo package, and its `Modules()` returns both
|
||||||
|
- The default embed holds every Swedish postort the import can place, with all its streets, and, of the US places that hold most of a postal code's land, every state capital, every place of 100,000 or more and a population-weighted tenth, each with a quarter of its streets, at least ten
|
||||||
|
- A locale's `address` binds its country's record once and prints it whole
|
||||||
|
- A postort's kommun comes from its name, its tätort or its codes, never from distance
|
||||||
|
- A highway designation is not a street, and a US postal code belongs to the place holding most of its land inside places
|
||||||
|
- `--list` stays a plain list of paths
|
||||||
|
- A layout is always quoted
|
||||||
|
- A title is a table under `sex`
|
||||||
|
- A table owns the spelling of a selector on it
|
||||||
|
- No builtin reads the clock, so a date is bounded by days, never by an age
|
||||||
|
- `misc` is what every locale shares
|
||||||
|
- A register's canonical spelling loses to the one its domain writes
|
||||||
|
- `misc.territory` is the spine, and a `misc` table naming a territory links to it
|
||||||
|
- A table whose register publishes no frequency draws evenly
|
||||||
|
- `misc.timezone` weighs a zone by the people living in it
|
||||||
|
- `misc.car` is one flat table, not a make linked to its models
|
||||||
|
- `misc.territory` names its sovereign in a column, and there is no `misc.country` table
|
||||||
|
- `misc.loglevel` is a table keyed by POSIX's keyword, carrying the code
|
||||||
|
- `misc.tld` keys carry the leading dot, where other tables key on a bare code
|
||||||
|
- `misc.tld` is a table of its own, and `misc.territory.tld` stays a column
|
||||||
|
- `misc.territory` carries a currency code, it does not link to `misc.currency`
|
||||||
|
- An extension may name two media types
|
||||||
|
- The Swedish ids draw Skatteverket's test series
|
||||||
|
- The US given names come from a mirror of the SSA file
|
||||||
|
- `List` advertises direct descents only
|
||||||
|
- A path draws through its compiled steps
|
||||||
|
- A name lives in the category binding it, or in the repeat or choice item binding it, and is drawn on its first read
|
||||||
|
- comprehension floor: every dimension and the overall at 7.0 or above, from 0.2.0
|
||||||
|
- The template engine stays the root package until item 107; what reads no engine type, a table's rows included, sits in `internal/`
|
||||||
|
- Goal 5.1's `{…}` covers `{{`, `}}` and a lone `}`, so the goal names no escape
|
||||||
|
- Goal 7.2 holds while a first template can be written with no escape; a literal brace still takes `{{` or `}}`
|
||||||
|
- The locales are Ethnologue 2026's ten most-spoken languages, each in the country with the most speakers of it, the Nordic countries and Germany
|
||||||
|
- A country with no open address register covering it whole ships its places and postal codes without streets
|
||||||
|
- The apt, dnf and apk repositories are static files on GitHub Pages, holding the latest release only
|
||||||
|
- The image is pushed to GHCR and copied to Docker Hub, whose name the README shows
|
||||||
|
- A known vulnerability in the Go version or a required module blocks a release, even in code fejkdata never calls, and blocks no merge
|
||||||
|
|
||||||
|
# Scoring run
|
||||||
|
|
||||||
|
The project values for `~/.claude/skills/comprehension-panel/scoring-run.md`:
|
||||||
|
|
||||||
|
- `{language}`: Go
|
||||||
|
- `{kind}`: a Go library and CLI that renders fake data from JSON templates
|
||||||
|
- `{domain}`: fake test-data generation and template languages
|
||||||
|
- `{domain docs}`: the documentation of any fake-data library, such as Faker
|
||||||
|
- `{3am question}`: a seeded render of `sv_SE.address` prints the postal code of one postort beside a street of another
|
||||||
|
|||||||
+96
-82
@@ -8,85 +8,99 @@ replacement, and each removed path, column or flag.
|
|||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
- First release: the CLI, the library and the shipped data set.
|
- Prebuilt binaries for Linux, macOS and Windows on amd64 and arm64, packages for apt,
|
||||||
- Table categories: a category JSON naming a `rows` TSV beside it, with the options
|
dnf, apk, Homebrew and Scoop, and the image `larvit/fejkdata` on Docker Hub, also at `ghcr.io/larvit/fejkdata`, so
|
||||||
`key`, `name`, `weight` and `parent`; a path selects a row by key or name,
|
the CLI installs without Go. 0.1.0 ships them too; the README's
|
||||||
`misc.territory[SE]`, and descends to a linked table by name; linked tables draw
|
[Install](https://github.com/larvit/fejkdata#install) lists each command.
|
||||||
consistently within one render and draw group. `rows` is an option, so no
|
|
||||||
template may carry a field of that name. A `name` without a `key` resolves
|
## [0.1.0] - 2026-10-13
|
||||||
inside the table's `parent`. Refused at `New`: a `name` without a `key` or a
|
|
||||||
`parent`, a name repeating inside one parent row, a name spelling another row's key, a table named like a column of any
|
The first release: a Go library and a CLI that render locale-aware fake data from JSON
|
||||||
table above it, a table whose format or cell references its own family, and,
|
templates, and the data they ship with. The
|
||||||
within one render and draw group, a path drawing a table another path selects a
|
[README](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md) documents each part.
|
||||||
row of, or two paths pinning different rows of one table.
|
Install the CLI with `go install github.com/larvit/fejkdata/cmd/fejkdata@v0.1.0`, or the
|
||||||
- `New` refuses a root choice of templates sharing one format and one set of string
|
library with `go get github.com/larvit/fejkdata@v0.1.0`. Before 1.0.0 a minor may break,
|
||||||
fields, naming the rows TSV to write instead.
|
and any release may change what a seed draws, so pin one exact version
|
||||||
- `misc.territory`, `misc.currency`, `misc.language`, `misc.httpstatus` and
|
([Versioning](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#versioning)).
|
||||||
`misc.mimetype` are tables. `misc.territory` is every ISO 3166-1 territory that has
|
|
||||||
a capital, a currency and a TLD, with the columns `calling-code`, `capital`,
|
### Added
|
||||||
`country`, `currency`, `flag`, `languages`, `numeric` and `tld` added, `country`
|
|
||||||
holding the alpha-2 code of the sovereign state it belongs to and its own where it
|
- The CLI, `fejkdata`, renders exactly the template on its stdin and adds nothing:
|
||||||
is one; `misc.currency` the
|
`echo '{/sv_SE.person} hihi' | fejkdata` prints e.g. `Sara Eriksson hihi`. `--seed`
|
||||||
current ISO 4217 currencies with a minor unit, with `decimals` and `numeric`
|
repeats a run's output while the version and data stay the same, and without it each
|
||||||
added, and its symbols from CLDR; `misc.language` every ISO 639-2 entry carrying a
|
run takes a new seed; `-n` renders the template that many times, and `--list`
|
||||||
639-1 code, with a `code3` column holding its 639-2/T code; `misc.httpstatus` each
|
prints the paths the data offers.
|
||||||
code the IANA registry lists with a plain reason phrase; and `misc.mimetype` each
|
`--data-path` layers a directory of your own data over the shipped set, and
|
||||||
type mime-db records as IANA-registered and gives a filename extension.
|
`--no-shipped-data` loads that directory alone. `--format json|ndjson|csv|sql` writes
|
||||||
`DATA-LICENSES.md` lists each source.
|
a record's columns, one record per row. The CLI exits `0` on success, `1` on a runtime
|
||||||
- `misc.timezone`, `misc.car` and `misc.useragent` are tables too. `misc.timezone`
|
error such as data that fails to load, and `2` on misuse.
|
||||||
is every zone tzdb `zone.tab` gives a shipped territory, with `offset` holding the
|
- The library, `github.com/larvit/fejkdata`, for Go 1.22 or later on a 64-bit target.
|
||||||
zone's standard UTC offset and `territory` linking to `misc.territory`, so
|
`New` loads the modules `WithDataFS` names and the folders `WithDataPath` names, drawing
|
||||||
`misc.territory[SE].timezone` draws `Europe/Stockholm`; zones of a territory
|
from the seed `WithSeed` gives. Without a seed, `New` fails and names `WithSeed`;
|
||||||
`misc.territory` does not ship, Antarctica's among them, and the constant `UTC`
|
without data, it fails and names `WithDataFS`. The same seed repeats the draws
|
||||||
are gone; spell that one as the text `"UTC"`. A `weight` column carries the
|
while the version and data stay the same. `Fake` renders a path, and `FakeTemplate`
|
||||||
population GeoNames records in each zone, so a draw favours the zones people live
|
and `NewTemplate` render a template. `FakeRecord`, `FakeRecordTemplate` and
|
||||||
in. `misc.useragent` is the top-user-agents desktop and mobile lists with `browser`,
|
`NewRecordTemplate` render a record, whose `JSON`, `CSVHeader`, `CSVLine` and
|
||||||
`device` and `os` columns. `misc.car` keeps its makes and models in a `make` and a
|
`SQLInsert` methods write its columns as the CLI does. `FakeStruct` fills a struct's
|
||||||
`model` column.
|
`fake:"…"` tagged fields. `List` gives the paths `Fake` renders, direct descents only,
|
||||||
- `misc.httpmethod`, `misc.protocol` and `misc.port` are IANA's registries as tables.
|
and `IsTemplate` tells an inline template from a path. Data that fails to load gives
|
||||||
`misc.httpmethod` is the eight methods RFC 9110 defines and PATCH, carrying
|
an error matching `ErrLoad`, and a record with no fields gives one matching
|
||||||
`safe` and `idempotent` as `true` or `false`; `misc.protocol` the keyword, its
|
`ErrNoColumns`.
|
||||||
`number` and its `name`, selectable by either; `misc.port` renders a TCP port number,
|
- Package `github.com/larvit/fejkdata/cli` is the `fejkdata` command:
|
||||||
with the IANA `service` name beside it, keeping the first service the registry
|
`cli.Main(modules...)` runs it over the modules given, so
|
||||||
describes for a port so a number selects one row.
|
`cli.Main(append(data.Modules(), mydata)...)` builds a CLI carrying your own data.
|
||||||
- `misc.tld` is the IANA root zone as a table: every delegated TLD keyed with its
|
That CLI names itself by the file it runs from, and its `--version` prints its own
|
||||||
leading dot, `.se` and `.xn--p1ai`, the register's `type` beside it, and a `unicode`
|
module's version before fejkdata's.
|
||||||
column holding the form the register displays, so `misc.tld[.рф]` selects the row
|
- The template language
|
||||||
`.xn--p1ai` renders. A delegation the register marks "Not assigned" does not ship.
|
([Data format](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data-format)).
|
||||||
- `misc.loglevel` is the eight syslog severities: the `keyword` a configuration writes,
|
Every value in a data file is a string, a choice, a template, a table, a number or a
|
||||||
`err` and `info`, keyed by the numerical `code` a PRI encodes and selectable by
|
boolean. A `{…}` token reads a sibling field, a name, or a reference from the data
|
||||||
either, with the `severity` RFC 5424 spells beside it.
|
root, the file's folder or the folder above (`{/sv_SE.person.last}`, `{.x}`, `{..x}`),
|
||||||
- `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`,
|
or it calls a builtin. Every `{…}` draws afresh, and a name is the one way to keep a
|
||||||
`locality`, `postal-code` and `street`, weighted by population and address counts
|
pick: `{/sv_SE.person as p}{p.first} {p.last}` is one person. The builtins draw check
|
||||||
and built from SCB, GeoNames, Trafikverket NVDB and the US Census Bureau, and an
|
digits (`luhn`, `mod11`, `ean`), samples (`digits`, `int`, `float`, `uuid`, `ulid`,
|
||||||
`address` record over one consistent draw of them. `sv_SE.address` and
|
`iban`, `date`, `time` and more), a counter (`seq`), arithmetic (`calc`) and
|
||||||
`en_US.address` read those records, so `en_US.address.street` no longer carries
|
transforms (`lowercase`, `uppercase`, `ascii`). `New` refuses data that breaks the
|
||||||
`name` and `suffix`, and a locale folder loads only beside `geo`.
|
grammar, could mean two things or cannot render a valid value.
|
||||||
- `{date(from,to,'layout')}` and `{time('layout')}`: a second between two days, or
|
- Records ([Records](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#records)):
|
||||||
within one, in a single-quoted Go layout, drawn in UTC; `from` may equal `to`.
|
a template seen as columns. A column is a string unless it declares a `datatype`,
|
||||||
`sv_SE.date`, `en_US.date`, `sv_SE.time` and `en_US.time` render through them, so
|
`integer`, `number` or `boolean`, and a `null` item draws it as null.
|
||||||
`date.year`, `date.month`, `date.day`, `time.hour`, `time.minute`, `time.minute.t`,
|
- Tables ([Table](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#table)): a
|
||||||
`time.sec` and `time.ampm` are no longer paths — a part of a date is now its own
|
category naming a `rows` TSV beside it, with the options `key`, `name`, `weight` and
|
||||||
`{date(…,'2006')}`. `misc.datetime` is an RFC 3339 instant.
|
`parent`. A path selects a row by key or name, `misc.territory[SE]`, descends to a
|
||||||
- `sex`, `first-name` and `last-name` tables in `sv_SE` and `en_US`, weighted by
|
linked table, `geo.SE.locality[Lund].street`, and steps up to the parent row,
|
||||||
bearers from SCB, the SSA and the Census Bureau; `first-name` links to `sex`, and a
|
`{/misc.timezone[Europe/Oslo]..territory.name}`.
|
||||||
name both sexes carry is a row under each. `person` reads them, so its columns are
|
- Data sources ([Data](https://github.com/larvit/fejkdata/blob/v0.1.0/README.md#data)):
|
||||||
`first`, `last`, `prefix` and `sex`; `person.femalefirst` and `person.malefirst` are
|
a directory is a namespace, and each JSON file in it a category. A `.fejkdata.json`
|
||||||
no longer paths — draw `sex[f].first-name` and `sex[m].first-name` instead.
|
manifest at a source's root may carry a `namespace`, the dot path the source mounts
|
||||||
`en_US.title` links to `sex` as well, so `en_US.person.prefix` draws `Mr` or `Ms`
|
under, and an `index`, with which each category loads on the first call reaching it.
|
||||||
without contradicting the record's `sex`; `sv_SE.title` is an unsexed table of the
|
- The shipped data, in Go packages that each export `FS` and `Modules()`:
|
||||||
same shape.
|
`github.com/larvit/fejkdata/data/en_US` (package `enus`), `…/sv_SE` (`svse`),
|
||||||
- `sv_SE.personnummer` and `sv_SE.samordningsnummer`, Skatteverket's test series
|
`…/misc`, `…/geo/SE` (`geose`) and `…/geo/US` (`geous`). `data.Modules()`, from
|
||||||
from a `sv_SE.birth-number` table under `sex`, in place of `sv_SE.ssn`, whose
|
`github.com/larvit/fejkdata/data`, lists every one, and the CLI carries them all.
|
||||||
`ssn.mmdd`, `ssn.mmdd.m` and `ssn.mmdd.d` go with it. `en_US.ssn` now draws the
|
[`DATA-LICENSES.md`](https://github.com/larvit/fejkdata/blob/v0.1.0/DATA-LICENSES.md)
|
||||||
ranges the SSA assigns and carries the columns `area`, `group` and `serial`, so
|
names each source.
|
||||||
`--format csv en_US.ssn` writes a header where it used to fail; `en_US.itin` is new
|
- `sv_SE` and `en_US`: an `address`, a `person` and more per locale. `sex`,
|
||||||
and carries the same three. `person.prefix` is null where a person has no title,
|
`first-name` and `last-name` are tables weighted by bearers, from SCB, the SSA and
|
||||||
where it used to be an empty string, so `--format sql` writes `NULL` and a `string`
|
the Census Bureau. A person's first name and sex agree, and so does an `en_US`
|
||||||
struct field reading it becomes `*string`.
|
person's title; `person.prefix` is null where a person has no title.
|
||||||
- `ErrNoColumns` is exported, so a caller can tell the one record fence a path can
|
`sv_SE.personnummer` and `sv_SE.samordningsnummer` draw Skatteverket's test series.
|
||||||
answer from the rest.
|
`en_US.ssn` and `en_US.itin` draw the ranges the SSA and the IRS assign, with the
|
||||||
- An error names a spelling that runs: a layout is named single-quoted and free of
|
columns `area`, `group` and `serial`, and `en_US.phone` draws an exchange NANPA may
|
||||||
its own quotes, a row of a table with no key is named as the path that selects it,
|
assign.
|
||||||
`sv_SE.sex[f].first-name[Kim]`, and a category with no columns names the record
|
- `misc`: IANA's registries of HTTP methods, protocols, ports and root-zone TLDs, the
|
||||||
that gives it one.
|
HTTP status codes, ISO 3166-1 territories, ISO 4217 currencies with their CLDR
|
||||||
|
symbols, ISO 639 languages, the syslog severities, mime-db's media types, tzdb's
|
||||||
|
zones weighted by the people living in each, user agents and cars, all as tables;
|
||||||
|
and coordinates, credit card numbers, RFC 3339 instants, emoji, MAC addresses,
|
||||||
|
ObjectIds and UUIDs.
|
||||||
|
- `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`,
|
||||||
|
`locality`, `postal-code` and `street`, weighted by population and address counts
|
||||||
|
and built from SCB, GeoNames, Trafikverket NVDB, the US Census Bureau and Wikidata,
|
||||||
|
and an `address` record that draws a street and a postal code inside one
|
||||||
|
locality; `sv_SE.address` and `en_US.address` print it. Every Swedish postort
|
||||||
|
(postal town) the import can place ships with all its streets. A US place ships
|
||||||
|
only if it is incorporated and holds most of some postal code's land. Of those
|
||||||
|
places, every state capital but Honolulu ships, as does every place of 100,000
|
||||||
|
people or more and a tenth of all of them drawn by population. Each US place
|
||||||
|
carries a quarter of its streets, at least ten.
|
||||||
|
|||||||
+2
-2
@@ -9,12 +9,12 @@ Every shipped dataset, its source, its licence and the attribution it asks for.
|
|||||||
| `geo/SE/locality.tsv`, `postal-code.tsv` | [GeoNames](https://www.geonames.org/) postal codes for SE; populations from SCB tätorter 2023 | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/); CC0 1.0 | "Postal codes from GeoNames, www.geonames.org" | `data-import/geo-se.py` |
|
| `geo/SE/locality.tsv`, `postal-code.tsv` | [GeoNames](https://www.geonames.org/) postal codes for SE; populations from SCB tätorter 2023 | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/); CC0 1.0 | "Postal codes from GeoNames, www.geonames.org" | `data-import/geo-se.py` |
|
||||||
| `geo/SE/region.tsv`, `municipality.tsv` | [SCB](https://www.scb.se/) län and kommun codes 2026 and population 2024 | [CC0 1.0](https://creativecommons.org/publicdomain/zero/1.0/) | none required | `data-import/geo-se.py` |
|
| `geo/SE/region.tsv`, `municipality.tsv` | [SCB](https://www.scb.se/) län and kommun codes 2026 and population 2024 | [CC0 1.0](https://creativecommons.org/publicdomain/zero/1.0/) | none required | `data-import/geo-se.py` |
|
||||||
| `geo/SE/street.tsv` | [Trafikverket NVDB](https://www.trafikverket.se/) Gatunamn, through the open API | CC0 1.0 | none required | `data-import/geo-se.py` |
|
| `geo/SE/street.tsv` | [Trafikverket NVDB](https://www.trafikverket.se/) Gatunamn, through the open API | CC0 1.0 | none required | `data-import/geo-se.py` |
|
||||||
| `geo/US/region.tsv`, `municipality.tsv`, `locality.tsv` | [Census Bureau](https://www.census.gov/) Gazetteer 2026 and population estimates 2025 | [public domain](https://www.usa.gov/government-works) | none required | `data-import/geo-us.py` |
|
| `geo/US/region.tsv`, `municipality.tsv`, `locality.tsv` | [Census Bureau](https://www.census.gov/) Gazetteer 2026 and population estimates 2025; which places are state capitals from [Wikidata](https://www.wikidata.org/) | [public domain](https://www.usa.gov/government-works); CC0 1.0 | none required | `data-import/geo-us.py` |
|
||||||
| `geo/US/postal-code.tsv`, `street.tsv` | Census Bureau ZCTA to place relationships 2020 and TIGER/Line 2025 address ranges and feature names | public domain | none required | `data-import/geo-us.py` |
|
| `geo/US/postal-code.tsv`, `street.tsv` | Census Bureau ZCTA to place relationships 2020 and TIGER/Line 2025 address ranges and feature names | public domain | none required | `data-import/geo-us.py` |
|
||||||
| `sv_SE/first-name.tsv`, `last-name.tsv` | [SCB](https://www.scb.se/) names with at least two bearers, 31 December 2022 | CC0 1.0 | "Källa: SCB" | `data-import/names-se.py` |
|
| `sv_SE/first-name.tsv`, `last-name.tsv` | [SCB](https://www.scb.se/) names with at least two bearers, 31 December 2022 | CC0 1.0 | "Källa: SCB" | `data-import/names-se.py` |
|
||||||
| `en_US/first-name.tsv` | [SSA](https://www.ssa.gov/oact/babynames/) baby names, births 1930 to 2020, through [hackerb9/ssa-baby-names](https://github.com/hackerb9/ssa-baby-names) | public domain | none required | `data-import/names-us.py` |
|
| `en_US/first-name.tsv` | [SSA](https://www.ssa.gov/oact/babynames/) baby names, births 1930 to 2020, through [hackerb9/ssa-baby-names](https://github.com/hackerb9/ssa-baby-names) | public domain | none required | `data-import/names-us.py` |
|
||||||
| `en_US/last-name.tsv` | Census Bureau surnames occurring 100 or more times, 2010 | public domain | none required | `data-import/names-us.py` |
|
| `en_US/last-name.tsv` | Census Bureau surnames occurring 100 or more times, 2010 | public domain | none required | `data-import/names-us.py` |
|
||||||
| `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv`, `misc/car.tsv` | curated; Skatteverket's test birth numbers are facts, and `car.tsv` waits for an international source (goal 12) | — | — | — |
|
| `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv`, `misc/car.tsv` | curated; Skatteverket's test birth numbers are facts, and `car.tsv` waits for an international source (goal 13) | — | — | — |
|
||||||
| `misc/currency.tsv` | [datasets/currency-codes](https://github.com/datasets/currency-codes); symbols from [Unicode CLDR](https://github.com/unicode-org/cldr) `en.xml` and `root.xml` | PDDL 1.0; [Unicode License v3](https://www.unicode.org/license.txt) | CLDR: "Copyright © 1991-2025 Unicode, Inc. Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries." | `data-import/currency.py` |
|
| `misc/currency.tsv` | [datasets/currency-codes](https://github.com/datasets/currency-codes); symbols from [Unicode CLDR](https://github.com/unicode-org/cldr) `en.xml` and `root.xml` | PDDL 1.0; [Unicode License v3](https://www.unicode.org/license.txt) | CLDR: "Copyright © 1991-2025 Unicode, Inc. Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries." | `data-import/currency.py` |
|
||||||
| `misc/httpmethod.tsv` | [IANA HTTP Method Registry](https://www.iana.org/assignments/http-methods/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpmethod.py` |
|
| `misc/httpmethod.tsv` | [IANA HTTP Method Registry](https://www.iana.org/assignments/http-methods/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpmethod.py` |
|
||||||
| `misc/httpstatus.tsv` | [IANA HTTP Status Code Registry](https://www.iana.org/assignments/http-status-codes/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpstatus.py` |
|
| `misc/httpstatus.tsv` | [IANA HTTP Status Code Registry](https://www.iana.org/assignments/http-status-codes/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpstatus.py` |
|
||||||
|
|||||||
+33
-9
@@ -1,19 +1,43 @@
|
|||||||
# The merge gate: `docker build .` fails if vet, formatting or tests fail, and CI
|
ARG GO_VERSION=1.27.2
|
||||||
# runs exactly this. Day-to-day, prefer `docker compose run` (bind-mounts source,
|
FROM golang:${GO_VERSION} AS portable
|
||||||
# no rebuilds). GO_VERSION defaults to the latest stable Go; override it to test
|
|
||||||
# the lowest supported version: docker build --build-arg GO_VERSION=1.22.12 .
|
|
||||||
ARG GO_VERSION=1.26.4
|
|
||||||
FROM golang:${GO_VERSION}
|
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
# Module layer cached separately from source.
|
|
||||||
COPY go.mod ./
|
COPY go.mod ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN go vet ./... && \
|
RUN go vet ./... && \
|
||||||
go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test . && \
|
go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test . && \
|
||||||
{ unformatted="$(gofmt -l .)"; test -z "$unformatted" || \
|
|
||||||
{ echo "unformatted files:"; echo "$unformatted"; exit 1; }; } && \
|
|
||||||
go test -race ./...
|
go test -race ./...
|
||||||
|
|
||||||
|
FROM golang:${GO_VERSION} AS vulnerabilities
|
||||||
|
WORKDIR /app
|
||||||
|
COPY . .
|
||||||
|
RUN go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan module -format json > /report.json
|
||||||
|
|
||||||
|
FROM scratch AS vulnerability-report
|
||||||
|
COPY --from=vulnerabilities /report.json /
|
||||||
|
|
||||||
|
FROM python:3.14.8-slim AS release-tooling
|
||||||
|
COPY release-tooling/ /release-tooling/
|
||||||
|
RUN python3 -m unittest discover -s /release-tooling -p '*_test.py' && touch /tested
|
||||||
|
|
||||||
|
FROM python:3.14.8-slim AS vulnerability-filing
|
||||||
|
COPY vulnerability-filing/ /vulnerability-filing/
|
||||||
|
# The filer's test files an item into the plan itself.
|
||||||
|
COPY todo.md /todo.md
|
||||||
|
RUN python3 -m unittest discover -s /vulnerability-filing -p '*_test.py' && touch /tested
|
||||||
|
|
||||||
|
FROM goreleaser/goreleaser:v2.18.3 AS packaging
|
||||||
|
WORKDIR /check
|
||||||
|
COPY release-tooling/goreleaser.yaml ./
|
||||||
|
RUN git init -q && git remote add origin https://github.com/larvit/fejkdata.git && goreleaser check --config goreleaser.yaml && touch /tested
|
||||||
|
|
||||||
|
FROM portable
|
||||||
|
# BuildKit skips a stage nothing copies from.
|
||||||
|
COPY --from=release-tooling /tested /tmp/release-tooling-tested
|
||||||
|
COPY --from=vulnerability-filing /tested /tmp/vulnerability-filing-tested
|
||||||
|
COPY --from=packaging /tested /tmp/packaging-tested
|
||||||
|
RUN unformatted="$(gofmt -l .)"; test -z "$unformatted" || \
|
||||||
|
{ echo "unformatted files:"; echo "$unformatted"; exit 1; }
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
MIT License
|
MIT License
|
||||||
|
|
||||||
Copyright (c) 2026 Timewave
|
Copyright (c) 2026 Timewave
|
||||||
|
Copyright (c) 2026 Larvit
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
|||||||
@@ -0,0 +1,117 @@
|
|||||||
|
package fejkdata
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata/internal/builtinfunc"
|
||||||
|
"github.com/larvit/fejkdata/internal/grammar"
|
||||||
|
"github.com/larvit/fejkdata/internal/invariant"
|
||||||
|
)
|
||||||
|
|
||||||
|
// arm is one alternative of a {a|b} token or one operand, split into the head,
|
||||||
|
// whose node `template.startOf` finds, and the tail of a dotted path into it.
|
||||||
|
type arm struct {
|
||||||
|
spelling string // as written, for messages
|
||||||
|
head string
|
||||||
|
tail []string
|
||||||
|
levels []pickKey // the key of each level the steps pass, from the head they start at to the leaf; a step's at indexes it
|
||||||
|
steps []pathStep
|
||||||
|
leaves []node // every node the path may land on, one per variant it passes
|
||||||
|
kind armKind
|
||||||
|
named *nameBinding // namedRead: the binding of the name it reads through
|
||||||
|
}
|
||||||
|
|
||||||
|
// armKind is how expand reads an arm, fixed at compile.
|
||||||
|
type armKind uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
freshRead armKind = iota // drawn afresh at every read
|
||||||
|
namedRead // read through a name, kept in its pick
|
||||||
|
)
|
||||||
|
|
||||||
|
// splitArm splits one name into head and tail. refs maps a reference to what
|
||||||
|
// resolveRefs resolved it to; before that, a reference is whole.
|
||||||
|
func splitArm(name string, refs map[string]resolvedRef) arm {
|
||||||
|
if grammar.IsRef(name) {
|
||||||
|
if r, resolved := refs[name]; resolved {
|
||||||
|
return pathArm(name, r.head, r.tail)
|
||||||
|
}
|
||||||
|
return pathArm(name, name, nil)
|
||||||
|
}
|
||||||
|
segs, err := grammar.SplitPath(name)
|
||||||
|
if err != nil || len(segs) == 1 {
|
||||||
|
return pathArm(name, name, nil)
|
||||||
|
}
|
||||||
|
return pathArm(name, segs[0], segs[1:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func pathArm(name, head string, segs []string) arm {
|
||||||
|
return arm{spelling: name, head: head, tail: segs, levels: levelKeys(append([]string{head}, segs...), 1)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// key is the key of the leaf a lands on, the one key every way of writing this read shares.
|
||||||
|
func (a arm) key() pickKey { return a.levels[len(a.levels)-1] }
|
||||||
|
|
||||||
|
func checkSegments(a arm) error {
|
||||||
|
if len(a.tail) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return grammar.CheckSegments(append([]string{a.head}, a.tail...))
|
||||||
|
}
|
||||||
|
|
||||||
|
// op is one compiled unit of a format string: a literal run, a name read,
|
||||||
|
// or a builtin already prepared with its args.
|
||||||
|
type op struct {
|
||||||
|
grammar.Token
|
||||||
|
arms []arm // grammar.PathRead: the '|' alternatives, split into head and tail once
|
||||||
|
call builtinfunc.Call
|
||||||
|
// operands are the fields the builtin reads, in the order its operands func
|
||||||
|
// fixed; expand reads them before the call. nil for a builtin that reads none.
|
||||||
|
operands []arm
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *template) compileArms(names []string, targets map[*nameBinding]nameTarget) ([]arm, error) {
|
||||||
|
if len(names) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
arms := make([]arm, len(names))
|
||||||
|
for i, name := range names {
|
||||||
|
a, err := t.compileArm(name, targets)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
arms[i] = a
|
||||||
|
}
|
||||||
|
return arms, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// compileArm compiles one read into a path: from the head it names, or, for a read through a
|
||||||
|
// name, from what that name binds.
|
||||||
|
func (t *template) compileArm(name string, targets map[*nameBinding]nameTarget) (arm, error) {
|
||||||
|
a := splitArm(name, t.refs.byName)
|
||||||
|
if !t.isName(a.head) {
|
||||||
|
start := t.startOf(a.head)
|
||||||
|
if start == nil {
|
||||||
|
panic(invariant.Broken("{%s} reads a head nothing resolved", a.spelling))
|
||||||
|
}
|
||||||
|
w := compilePath(start, a.tail)
|
||||||
|
a.steps, a.leaves = w.steps, w.leaves
|
||||||
|
return a, nil
|
||||||
|
}
|
||||||
|
b := t.nameScope.lookup(a.head)
|
||||||
|
if grammar.HasSelector(a.tail) {
|
||||||
|
if b.bindsField() {
|
||||||
|
return a, fmt.Errorf("a path through name %q may not select a row; bind the row to a name of its own", a.head)
|
||||||
|
}
|
||||||
|
return a, fmt.Errorf("a path through name %q may not select a row; read the row without the name, {%s}, or bind it to a name of its own", a.head, grammar.JoinSegments(append([]string{b.ref}, a.tail...)))
|
||||||
|
}
|
||||||
|
target := targets[b]
|
||||||
|
full := append(target.tail[:len(target.tail):len(target.tail)], a.tail...)
|
||||||
|
if err := provePath(target.start, full, a.head, len(target.tail)); err != nil {
|
||||||
|
return a, err
|
||||||
|
}
|
||||||
|
w := compilePath(target.start, full)
|
||||||
|
a.kind, a.named, a.steps, a.leaves = namedRead, b, w.steps, w.leaves
|
||||||
|
a.levels = levelKeys(full, 0)
|
||||||
|
return a, nil
|
||||||
|
}
|
||||||
+44
-32
@@ -6,17 +6,17 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
func benchGenerator(b *testing.B, dir string) *Generator {
|
func benchGenerator(b *testing.B, data Option) *Generator {
|
||||||
b.Helper()
|
b.Helper()
|
||||||
f, err := New(WithoutShippedData(), WithDataPath(dir), WithSeed(1))
|
f, err := New(data, WithSeed(1))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.Fatal(err)
|
b.Fatal(err)
|
||||||
}
|
}
|
||||||
return f
|
return f
|
||||||
}
|
}
|
||||||
|
|
||||||
func benchPath(b *testing.B, dir, path string) {
|
func benchPath(b *testing.B, data Option, path string) {
|
||||||
f := benchGenerator(b, dir)
|
f := benchGenerator(b, data)
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
b.ResetTimer()
|
b.ResetTimer()
|
||||||
for i := 0; i < b.N; i++ {
|
for i := 0; i < b.N; i++ {
|
||||||
@@ -26,12 +26,12 @@ func benchPath(b *testing.B, dir, path string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func BenchmarkPerson(b *testing.B) { benchPath(b, "data", "sv_SE.person") }
|
func BenchmarkPerson(b *testing.B) { benchPath(b, withShipped(), "sv_SE.person") }
|
||||||
func BenchmarkAddress(b *testing.B) { benchPath(b, "data", "sv_SE.address") }
|
func BenchmarkAddress(b *testing.B) { benchPath(b, withShipped(), "sv_SE.address") }
|
||||||
func BenchmarkWord(b *testing.B) { benchPath(b, "data", "sv_SE.word") }
|
func BenchmarkWord(b *testing.B) { benchPath(b, withShipped(), "sv_SE.word") }
|
||||||
func BenchmarkCreditcard(b *testing.B) { benchPath(b, "data", "misc.creditcard") }
|
func BenchmarkCreditcard(b *testing.B) { benchPath(b, withShipped(), "misc.creditcard") }
|
||||||
func BenchmarkPersonnummer(b *testing.B) { benchPath(b, "data", "sv_SE.personnummer") }
|
func BenchmarkPersonnummer(b *testing.B) { benchPath(b, withShipped(), "sv_SE.personnummer") }
|
||||||
func BenchmarkUUIDv7(b *testing.B) { benchPath(b, "data", "misc.uuid") }
|
func BenchmarkUUIDv7(b *testing.B) { benchPath(b, withShipped(), "misc.uuid") }
|
||||||
|
|
||||||
func tmpData(b *testing.B, name, body string) string {
|
func tmpData(b *testing.B, name, body string) string {
|
||||||
b.Helper()
|
b.Helper()
|
||||||
@@ -44,52 +44,64 @@ func tmpData(b *testing.B, name, body string) string {
|
|||||||
|
|
||||||
func BenchmarkCalc(b *testing.B) {
|
func BenchmarkCalc(b *testing.B) {
|
||||||
dir := tmpData(b, "inv", `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`)
|
dir := tmpData(b, "inv", `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`)
|
||||||
benchPath(b, dir, "inv")
|
benchPath(b, WithDataPath(dir), "inv")
|
||||||
}
|
}
|
||||||
|
|
||||||
func BenchmarkLongLiteral(b *testing.B) {
|
func BenchmarkLongLiteral(b *testing.B) {
|
||||||
dir := tmpData(b, "sql", `{"format":"INSERT INTO customers (id, name, city) VALUES (123, '{word}', '{word}');","word":["alpha","beta","gamma","delta"]}`)
|
dir := tmpData(b, "sql", `{"format":"INSERT INTO customers (id, name, city) VALUES (123, '{word}', '{word}');","word":["alpha","beta","gamma","delta"]}`)
|
||||||
benchPath(b, dir, "sql")
|
benchPath(b, WithDataPath(dir), "sql")
|
||||||
}
|
}
|
||||||
|
|
||||||
func BenchmarkRepeat(b *testing.B) {
|
func BenchmarkRepeat(b *testing.B) {
|
||||||
dir := tmpData(b, "many", `{"format":"{word}","repeat":20,"separator":", ","word":["alpha","beta","gamma","delta"]}`)
|
dir := tmpData(b, "many", `{"format":"{word}","repeat":20,"separator":", ","word":["alpha","beta","gamma","delta"]}`)
|
||||||
benchPath(b, dir, "many")
|
benchPath(b, WithDataPath(dir), "many")
|
||||||
}
|
}
|
||||||
|
|
||||||
// BenchmarkBound measures a format that binds a head and reads two paths from it —
|
// BenchmarkNamed measures a format that binds a field to a name and reads two paths through
|
||||||
// what a correlated pair costs against BenchmarkUnbound, the same output drawn from
|
// it — what a correlated pair costs against BenchmarkUnnamed, the same output drawn from two
|
||||||
// two independent fields.
|
// independent fields.
|
||||||
func BenchmarkBound(b *testing.B) {
|
func BenchmarkNamed(b *testing.B) {
|
||||||
dir := tmpData(b, "addr", `{"format":"{place.postal-code} {place.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`)
|
dir := tmpData(b, "addr", `{"format":"{place as p}{p.postal-code} {p.locality}","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`)
|
||||||
benchPath(b, dir, "addr")
|
benchPath(b, WithDataPath(dir), "addr")
|
||||||
}
|
}
|
||||||
|
|
||||||
func BenchmarkUnbound(b *testing.B) {
|
func BenchmarkUnnamed(b *testing.B) {
|
||||||
dir := tmpData(b, "addr", `{"format":"{postal-code} {locality}","postal-code":["1{digits(2)} {digits(2)}","573 {digits(2)}"],"locality":["Stockholm","Tranås"]}`)
|
dir := tmpData(b, "addr", `{"format":"{postal-code} {locality}","postal-code":["1{digits(2)} {digits(2)}","573 {digits(2)}"],"locality":["Stockholm","Tranås"]}`)
|
||||||
benchPath(b, dir, "addr")
|
benchPath(b, WithDataPath(dir), "addr")
|
||||||
}
|
}
|
||||||
|
|
||||||
// BenchmarkBoundDeep reads two paths through an intermediate level, the shape the
|
// BenchmarkNamedDeep reads two paths through a name and an intermediate level.
|
||||||
// depth question is about.
|
func BenchmarkNamedDeep(b *testing.B) {
|
||||||
func BenchmarkBoundDeep(b *testing.B) {
|
dir := tmpData(b, "addr", `{"format":"{p as q}{q.addr.city} {q.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`)
|
||||||
dir := tmpData(b, "addr", `{"format":"{p.addr.city} {p.addr.zip}","p":{"format":"{addr}","addr":{"format":"{city}","city":["Stockholm","Tranås"],"zip":"1{digits(2)} {digits(2)}"}}}`)
|
benchPath(b, WithDataPath(dir), "addr")
|
||||||
benchPath(b, dir, "addr")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// BenchmarkBoundWide reads ten paths from one row: the point where the maps an
|
// BenchmarkNamedWide reads ten paths through one name: the point where the maps a pick keeps
|
||||||
// expansion holds outgrow a single bucket.
|
// outgrow a single bucket.
|
||||||
func BenchmarkBoundWide(b *testing.B) {
|
func BenchmarkNamedWide(b *testing.B) {
|
||||||
dir := tmpData(b, "row", `{"format":"{r.a}{r.b}{r.c}{r.d}{r.e}{r.f}{r.g}{r.h}{r.i}{r.j}","r":[
|
dir := tmpData(b, "row", `{"format":"{r as s}{s.a}{s.b}{s.c}{s.d}{s.e}{s.f}{s.g}{s.h}{s.i}{s.j}","r":[
|
||||||
{"format":"x","a":"1","b":"2","c":"3","d":"4","e":"5","f":"6","g":"7","h":"8","i":"9","j":"0"},
|
{"format":"x","a":"1","b":"2","c":"3","d":"4","e":"5","f":"6","g":"7","h":"8","i":"9","j":"0"},
|
||||||
{"format":"y","a":"A","b":"B","c":"C","d":"D","e":"E","f":"F","g":"G","h":"H","i":"I","j":"J"}]}`)
|
{"format":"y","a":"A","b":"B","c":"C","d":"D","e":"E","f":"F","g":"G","h":"H","i":"I","j":"J"}]}`)
|
||||||
benchPath(b, dir, "row")
|
benchPath(b, WithDataPath(dir), "row")
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkFirstFake(b *testing.B) {
|
||||||
|
b.ReportAllocs()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
f, err := New(withShipped(), WithSeed(1))
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := f.Fake("sv_SE.address"); err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func BenchmarkNew(b *testing.B) {
|
func BenchmarkNew(b *testing.B) {
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
for i := 0; i < b.N; i++ {
|
for i := 0; i < b.N; i++ {
|
||||||
if _, err := New(WithSeed(1)); err != nil {
|
if _, err := New(withShipped(), WithSeed(1)); err != nil {
|
||||||
b.Fatal(err)
|
b.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+99
-51
@@ -2,11 +2,14 @@ package fejkdata
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata/internal/invariant"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBuiltinIDGenerators(t *testing.T) {
|
func TestBuiltinIDGenerators(t *testing.T) {
|
||||||
@@ -101,10 +104,10 @@ func TestBuiltinChecksums(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestBuiltinSeqPerSession pins seq's contract: a counter from 1, advancing on
|
// TestBuiltinSeqPerGenerator pins seq's contract: a counter from 1, advancing on
|
||||||
// each call, named counters independent, and the whole thing scoped to one
|
// each call, named counters independent, and the whole thing scoped to one
|
||||||
// session so a fresh Generator restarts at 1.
|
// draw state so a fresh Generator restarts at 1.
|
||||||
func TestBuiltinSeqPerSession(t *testing.T) {
|
func TestBuiltinSeqPerGenerator(t *testing.T) {
|
||||||
f := engine(1)
|
f := engine(1)
|
||||||
for i := 1; i <= 5; i++ {
|
for i := 1; i <= 5; i++ {
|
||||||
if got := mustRender(t, f, `"{seq()}"`); got != strconv.Itoa(i) {
|
if got := mustRender(t, f, `"{seq()}"`); got != strconv.Itoa(i) {
|
||||||
@@ -121,9 +124,9 @@ func TestBuiltinSeqPerSession(t *testing.T) {
|
|||||||
if got := mustRender(t, engine(2), `{"format":"{seq()}","repeat":3,"separator":","}`); got != "1,2,3" {
|
if got := mustRender(t, engine(2), `{"format":"{seq()}","repeat":3,"separator":","}`); got != "1,2,3" {
|
||||||
t.Fatalf("repeat seq = %q, want 1,2,3", got)
|
t.Fatalf("repeat seq = %q, want 1,2,3", got)
|
||||||
}
|
}
|
||||||
// ...and a fresh session restarts from 1.
|
// ...and a fresh generator restarts from 1.
|
||||||
if got := mustRender(t, engine(2), `"{seq()}"`); got != "1" {
|
if got := mustRender(t, engine(2), `"{seq()}"`); got != "1" {
|
||||||
t.Fatalf("new session seq = %q, want 1", got)
|
t.Fatalf("new generator seq = %q, want 1", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -142,7 +145,7 @@ func TestBuiltinArgLimits(t *testing.T) {
|
|||||||
`{"format":"{x}","x":"1","repeat":2000000000}`, // repeat -> multi-GB join
|
`{"format":"{x}","x":"1","repeat":2000000000}`, // repeat -> multi-GB join
|
||||||
}
|
}
|
||||||
for _, tmpl := range bad {
|
for _, tmpl := range bad {
|
||||||
if _, err := compile(parse(t, tmpl)); err == nil {
|
if _, err := resolved(t, tmpl); err == nil {
|
||||||
t.Errorf("compile(%s) = nil error, want a limit rejection", tmpl)
|
t.Errorf("compile(%s) = nil error, want a limit rejection", tmpl)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -187,51 +190,26 @@ func ibanValid(s string) bool {
|
|||||||
return rem == 1
|
return rem == 1
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRegistryShapes pins the builtin contract compileOps relies on: every entry
|
// mustPanic fails unless call panics with invariant.Broken's phrase, which is what
|
||||||
// supplies prep, and args parsed at compile only behind a check.
|
// separates a reported invariant break from a silently wrong value.
|
||||||
func TestRegistryShapes(t *testing.T) {
|
|
||||||
for name, b := range builtins {
|
|
||||||
if b.prep == nil {
|
|
||||||
t.Errorf("builtin %q has no prep: compileOps would call a nil func", name)
|
|
||||||
}
|
|
||||||
if b.arity != 0 && b.check == nil {
|
|
||||||
t.Errorf("builtin %q parses args in prep with no check", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArgGuardsPanic pins the guards that report a builtin arg its check should
|
|
||||||
// have rejected. No data reaches them — checkFunc runs a builtin's check before
|
|
||||||
// compileOps ever calls prep — so they are exercised directly.
|
|
||||||
func TestArgGuardsPanic(t *testing.T) {
|
|
||||||
for name, call := range map[string]func(){
|
|
||||||
"atoi on an unvalidated arg": func() { atoi("nope") },
|
|
||||||
"atof on an unvalidated arg": func() { atof("nope") },
|
|
||||||
} {
|
|
||||||
mustPanic(t, name, call)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// mustPanic fails unless call panics, which is what separates a reported
|
|
||||||
// invariant break from a silently wrong value.
|
|
||||||
func mustPanic(t *testing.T, name string, call func()) {
|
func mustPanic(t *testing.T, name string, call func()) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
defer func() {
|
defer func() {
|
||||||
if recover() == nil {
|
if r := recover(); r == nil || !strings.HasPrefix(fmt.Sprint(r), invariant.Broken("")) {
|
||||||
t.Errorf("%s: no panic, want the invariant reported", name)
|
t.Errorf("%s: recovered %v, want the invariant reported as an internal error", name, r)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
call()
|
call()
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestClassBuiltinArgs(t *testing.T) {
|
func TestClassBuiltinArgs(t *testing.T) {
|
||||||
for _, bad := range []string{`"{digits(0)}"`, `"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} {
|
for _, bad := range []string{`"{digits(2000000000)}"`, `"{upper(-1)}"`, `"{lower(x)}"`, `"{digits()}"`, `"{upper(1,2)}"`} {
|
||||||
if _, err := compile(parse(t, bad)); err == nil {
|
if _, err := resolved(t, bad); err == nil {
|
||||||
t.Errorf("compile(%s) = nil error, want the arg rejected", bad)
|
t.Errorf("compile(%s) = nil error, want the arg rejected", bad)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, ok := range []string{`"{digits(1048576)}"`, `"{upper(1)}"`, `"{lower(26)}"`} {
|
for _, ok := range []string{`"{digits(1048576)}"`, `"{upper(1)}"`, `"{lower(26)}"`} {
|
||||||
if _, err := compile(parse(t, ok)); err != nil {
|
if _, err := resolved(t, ok); err != nil {
|
||||||
t.Errorf("compile(%s) = %v", ok, err)
|
t.Errorf("compile(%s) = %v", ok, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -293,27 +271,22 @@ func TestBuiltinDateAndTime(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestBuiltinDateArgs pins the New-time checks: bounds are calendar dates in order,
|
// TestBuiltinDateArgs pins the New-time checks: bounds are calendar dates in order, and the
|
||||||
// the layout is quoted, names a field, and for time names no date field.
|
// layout is quoted.
|
||||||
func TestBuiltinDateArgs(t *testing.T) {
|
func TestBuiltinDateArgs(t *testing.T) {
|
||||||
for tmpl, want := range map[string]string{
|
for tmpl, want := range map[string]string{
|
||||||
`"{date(1990-13-01,1990-12-31,'2006-01-02')}"`: "1990-13-01",
|
`"{date(1990-13-01,1990-12-31,'2006-01-02')}"`: "1990-13-01",
|
||||||
|
`"{time('2006-01-02 15:04')}"`: "write date(from,to,layout)",
|
||||||
`"{date(1990-12-31,1990-01-01,'2006-01-02')}"`: "is after",
|
`"{date(1990-12-31,1990-01-01,'2006-01-02')}"`: "is after",
|
||||||
`"{date(1990-01-01,1990-01-01,'2006-01-02')}"`: "write it as text",
|
|
||||||
`"{date(1990-01-01,1990-12-31,2006-01-02)}"`: "'2006-01-02'",
|
`"{date(1990-01-01,1990-12-31,2006-01-02)}"`: "'2006-01-02'",
|
||||||
`"{date(1990-01-01,1990-12-31,'January 2, 2006)}"`: "'",
|
`"{date(1990-01-01,1990-12-31,'January 2, 2006)}"`: "'",
|
||||||
`"{date(1990-01-01,1990-12-31,\"January 2, 2006\")}"`: "quoted: 'January 2, 2006'",
|
`"{date(1990-01-01,1990-12-31,\"January 2, 2006\")}"`: "quoted: 'January 2, 2006'",
|
||||||
`"{date(1990-01-01,1990-12-31,'x')}"`: "text",
|
|
||||||
`"{date(1990-01-01,1990-12-31,'')}"`: "text",
|
|
||||||
`"{date(1990-01-01,1990-12-31)}"`: "3 arguments",
|
`"{date(1990-01-01,1990-12-31)}"`: "3 arguments",
|
||||||
`"{date(1990-01-01,1990-12-31,January 2, 2006)}"`: "'January 2, 2006'",
|
`"{date(1990-01-01,1990-12-31,January 2, 2006)}"`: "'January 2, 2006'",
|
||||||
`"{time(3:04 PM, Mon)}"`: "'3:04 PM, Mon'",
|
`"{time(3:04 PM, Mon)}"`: "'3:04 PM, Mon'",
|
||||||
`"{time(15:04)}"`: "'15:04'",
|
`"{time(15:04)}"`: "'15:04'",
|
||||||
`"{time('2006-01-02 15:04')}"`: "date(",
|
|
||||||
`"{time('x')}"`: "text",
|
|
||||||
`"{date(1990-01-01,1990-12-31,'15:04')}"`: "time('15:04')",
|
|
||||||
} {
|
} {
|
||||||
_, err := compile(parse(t, tmpl))
|
_, err := resolved(t, tmpl)
|
||||||
if err == nil || !strings.Contains(err.Error(), want) {
|
if err == nil || !strings.Contains(err.Error(), want) {
|
||||||
t.Errorf("compile(%s) = %v, want an error mentioning %q", tmpl, err, want)
|
t.Errorf("compile(%s) = %v, want an error mentioning %q", tmpl, err, want)
|
||||||
}
|
}
|
||||||
@@ -349,7 +322,7 @@ func TestBuiltinLayoutErrorsNameARunnableSpelling(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestBuiltinDateSpansOneDay pins that from == to is a day: with a clock layout it
|
// TestBuiltinDateSpansOneDay pins that from == to is a day: with a clock layout it
|
||||||
// draws every second of it, and without one it could only emit one value.
|
// draws every second of it, and without one it prints that day.
|
||||||
func TestBuiltinDateSpansOneDay(t *testing.T) {
|
func TestBuiltinDateSpansOneDay(t *testing.T) {
|
||||||
f := engine(1)
|
f := engine(1)
|
||||||
seen := map[string]bool{}
|
seen := map[string]bool{}
|
||||||
@@ -363,8 +336,83 @@ func TestBuiltinDateSpansOneDay(t *testing.T) {
|
|||||||
if len(seen) < 400 {
|
if len(seen) < 400 {
|
||||||
t.Fatalf("date over one day drew %d distinct seconds in 500, want the whole day", len(seen))
|
t.Fatalf("date over one day drew %d distinct seconds in 500, want the whole day", len(seen))
|
||||||
}
|
}
|
||||||
_, err := compile(parse(t, `"{date(2026-01-01,2026-01-01,'2006-01-02')}"`))
|
if got := mustRender(t, f, `"{date(2026-01-01,2026-01-01,'2006-01-02')}"`); got != "2026-01-01" {
|
||||||
if err == nil || !strings.Contains(err.Error(), "write it as text") {
|
t.Fatalf("one day in a date-only layout = %q, want that day", got)
|
||||||
t.Fatalf("one day in a date-only layout = %v, want it named a constant", err)
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClassBuiltins(t *testing.T) {
|
||||||
|
cases := map[string]*regexp.Regexp{
|
||||||
|
`"{digits(1)}"`: regexp.MustCompile(`^[0-9]$`),
|
||||||
|
`"{digits(3)}"`: regexp.MustCompile(`^[0-9]{3}$`),
|
||||||
|
`"{int(1,9)}"`: regexp.MustCompile(`^[1-9]$`),
|
||||||
|
`"{upper(1)}"`: regexp.MustCompile(`^[A-Z]$`),
|
||||||
|
`"{lower(1)}"`: regexp.MustCompile(`^[a-z]$`),
|
||||||
|
`"{upper(2)}{lower(2)}"`: regexp.MustCompile(`^[A-Z]{2}[a-z]{2}$`),
|
||||||
|
}
|
||||||
|
f := engine(7)
|
||||||
|
for tmpl, re := range cases {
|
||||||
|
for i := 0; i < 100; i++ {
|
||||||
|
if got := mustRender(t, f, tmpl); !re.MatchString(got) {
|
||||||
|
t.Fatalf("%s produced %q, want %s", tmpl, got, re)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFunctionTokenLuhn(t *testing.T) {
|
||||||
|
// {luhn()} appends a Luhn check digit over the digits emitted so far in the
|
||||||
|
// current expansion (non-digits skipped but kept). It reads the output
|
||||||
|
// buffer, so a value is never re-rendered. Bodies are escaped to fix input.
|
||||||
|
f := engine(1)
|
||||||
|
cases := map[string]string{
|
||||||
|
`"811218987{luhn()}"`: "8112189876", // personnummer body
|
||||||
|
`"7992739871{luhn()}"`: "79927398713", // classic Luhn vector
|
||||||
|
`"811218-987{luhn()}"`: "811218-9876", // '-' skipped, kept
|
||||||
|
`{"format":"{n}{luhn()}","n":"811218987"}`: "8112189876", // over a rendered token
|
||||||
|
}
|
||||||
|
for tmpl, want := range cases {
|
||||||
|
if got := mustRender(t, f, tmpl); got != want {
|
||||||
|
t.Fatalf("%s = %q, want %q", tmpl, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuiltinCompileErrors(t *testing.T) {
|
||||||
|
for _, bad := range []string{
|
||||||
|
`"{upper(x)}"`, // count must be an integer
|
||||||
|
`"{int(a,b)}"`, // non-integer args
|
||||||
|
`"{int(5,1)}"`, // min > max
|
||||||
|
`"{nanoid(-1)}"`, // negative count
|
||||||
|
`"{float(1,2,-1)}"`, // negative decimals
|
||||||
|
`"{float(NaN,NaN,2)}"`, // bounds must be finite
|
||||||
|
`"{float(Inf,Inf,2)}"`, // same-sign infinities
|
||||||
|
`"{float(1,NaN,2)}"`, // one NaN bound
|
||||||
|
`"{float(-Inf,1,2)}"`, // one infinite bound
|
||||||
|
`"{iban(US)}"`, // unsupported country
|
||||||
|
`"{seq(a,b)}"`, // seq takes at most one name
|
||||||
|
} {
|
||||||
|
if _, err := resolved(t, bad); err == nil {
|
||||||
|
t.Errorf("compile(%s) = nil error, want error", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArgErrorsNameTheSpelling(t *testing.T) {
|
||||||
|
if _, err := resolved(t, `"{hex(99999999999999999999)}"`); err == nil || !strings.Contains(err.Error(), "exceeds the maximum") {
|
||||||
|
t.Errorf("compile(hex past the range) = %v, want the maximum named", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestASignOrLeadingZeroIsTheInteger(t *testing.T) {
|
||||||
|
for long, short := range map[string]string{
|
||||||
|
`"{float(1,2,02)}"`: `"{float(1,2,2)}"`,
|
||||||
|
`{"format":"{calc(a,02)}","a":"1"}`: `{"format":"{calc(a,2)}","a":"1"}`,
|
||||||
|
`"{digits(+5)}"`: `"{digits(5)}"`,
|
||||||
|
`"{digits(05)}"`: `"{digits(5)}"`,
|
||||||
|
`"{int(007,9)}"`: `"{int(7,9)}"`,
|
||||||
|
`"{int(+1,5)}"`: `"{int(1,5)}"`,
|
||||||
|
} {
|
||||||
|
sameRenders(t, long, short)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,392 +2,149 @@ package fejkdata
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
"slices"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"unicode"
|
|
||||||
|
"github.com/larvit/fejkdata/internal/builtinfunc"
|
||||||
|
"github.com/larvit/fejkdata/internal/grammar"
|
||||||
|
"github.com/larvit/fejkdata/internal/invariant"
|
||||||
)
|
)
|
||||||
|
|
||||||
// calcNode is a parsed expression node. It evaluates over the operand values expand
|
// checkCalcFields runs checkOperands over each calc operand naming a field; an operand naming no
|
||||||
// read, so the evaluator touches neither the rng nor the node tree.
|
// field is a name, which checkCalcNames checks once names resolve.
|
||||||
type calcNode interface {
|
func checkCalcFields(args []string, fields map[string]node) error {
|
||||||
eval(operands []string) float64
|
return checkOperands(args[0], builtinfunc.ParsedCalc(args[0]), func(name string) []node {
|
||||||
}
|
if n, ok := fields[name]; ok {
|
||||||
|
return []node{n}
|
||||||
type calcNum float64 // a number literal
|
|
||||||
type calcVar string // a sibling-field name, before indexVars places it
|
|
||||||
type calcIdx int // an operand, by its position in the values expand read
|
|
||||||
type calcNeg struct{ x calcNode }
|
|
||||||
type calcBin struct { // a + - * / b
|
|
||||||
op byte
|
|
||||||
l, r calcNode
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n calcNum) eval([]string) float64 { return float64(n) }
|
|
||||||
|
|
||||||
func (n calcIdx) eval(operands []string) float64 {
|
|
||||||
v, err := strconv.ParseFloat(strings.TrimSpace(operands[n]), 64)
|
|
||||||
if err != nil {
|
|
||||||
return math.NaN() // a non-numeric operand stays visible, never an error
|
|
||||||
}
|
|
||||||
return v
|
|
||||||
}
|
|
||||||
|
|
||||||
// eval on an unplaced name cannot happen: calcPrep runs indexVars over every
|
|
||||||
// expression it compiles, so only a calcIdx reaches a render. It panics rather
|
|
||||||
// than returning NaN, so a node kind indexVars forgets is a stack trace and not a
|
|
||||||
// silently wrong number.
|
|
||||||
func (n calcVar) eval([]string) float64 {
|
|
||||||
panic(fmt.Sprintf("fejkdata: calc operand %q was never placed", string(n)))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n calcNeg) eval(operands []string) float64 { return -n.x.eval(operands) }
|
|
||||||
|
|
||||||
func (n calcBin) eval(operands []string) float64 {
|
|
||||||
l, r := n.l.eval(operands), n.r.eval(operands)
|
|
||||||
switch n.op {
|
|
||||||
case '+':
|
|
||||||
return l + r
|
|
||||||
case '-':
|
|
||||||
return l - r
|
|
||||||
case '*':
|
|
||||||
return l * r
|
|
||||||
default: // '/'
|
|
||||||
return l / r
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkCalc validates a calc token at compile time: a parseable expression whose
|
|
||||||
// operands all name existing fields, and an optional non-negative integer dp. It
|
|
||||||
// is a builtin check (fields first), so calc dispatches through the registry like
|
|
||||||
// every other {name(args)} function.
|
|
||||||
func checkCalc(fields map[string]node, args []string) error {
|
|
||||||
if len(args) < 1 || len(args) > 2 {
|
|
||||||
return fmt.Errorf("calc takes an expression and an optional decimals count, got %d args", len(args))
|
|
||||||
}
|
|
||||||
expr, err := parseCalc(args[0])
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("calc(%q): %w", args[0], err)
|
|
||||||
}
|
|
||||||
for _, name := range calcVars(expr) {
|
|
||||||
operand, ok := fields[name]
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("calc(%q): no field %q", args[0], name)
|
|
||||||
}
|
}
|
||||||
if text, never := neverNumeric(operand); never {
|
return nil
|
||||||
return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", args[0], name, text)
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkCalcNames holds each calc of t reading a name to the checks checkCalcFields makes of a field.
|
||||||
|
func checkCalcNames(label string, t *template) error {
|
||||||
|
for _, o := range t.compiled.ops {
|
||||||
|
if o.Fn != builtinfunc.CalcName || !slices.ContainsFunc(o.operands, func(a arm) bool { return a.kind == namedRead }) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := checkOperands(o.Args[0], builtinfunc.ParsedCalc(o.Args[0]), operandNodes(o)); err != nil {
|
||||||
|
return fmt.Errorf("%s: token {%s}: %w", label, o.Body, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if divisor, zero := constantZeroDivisor(expr, fields); zero {
|
return nil
|
||||||
return fmt.Errorf("calc(%q) divides by %s, which is always zero", args[0], divisor)
|
}
|
||||||
|
|
||||||
|
// operandNodes lists every node each operand of o may render, once its template and names resolve.
|
||||||
|
func operandNodes(o op) func(name string) []node {
|
||||||
|
return func(name string) []node {
|
||||||
|
a := o.operands[slices.IndexFunc(o.operands, func(a arm) bool { return a.head == name })]
|
||||||
|
if len(a.leaves) == 0 {
|
||||||
|
panic(invariant.Broken("calc operand %q was read before it was compiled", name))
|
||||||
|
}
|
||||||
|
return a.leaves
|
||||||
}
|
}
|
||||||
if len(args) == 2 {
|
}
|
||||||
dp, err := plainInt(args[1])
|
|
||||||
if err != nil {
|
// checkOperands refuses an operand that is never a number and a division by a constant zero.
|
||||||
return fmt.Errorf("calc decimals %w", err)
|
// operand lists every node an operand may render, nil while that is unknown.
|
||||||
}
|
func checkOperands(text string, c grammar.Calc, operand func(name string) []node) error {
|
||||||
if dp < 0 || dp > maxDecimals {
|
for _, name := range c.Operands {
|
||||||
return fmt.Errorf("calc decimals %d must be in 0..%d", dp, maxDecimals)
|
if rendered, never := allNeverNumeric(operand(name)); never {
|
||||||
|
return fmt.Errorf("calc(%q): operand %q is never a number: it renders %q", text, name, rendered)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if divisor, zero := constantZeroDivisor(c.Expr, operand); zero {
|
||||||
|
return fmt.Errorf("calc(%q) divides by %s, which is always zero", text, divisor)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// constantZeroDivisor finds a division whose right side is a constant zero: number
|
// constantZeroDivisor finds a division whose right side is a constant zero: number
|
||||||
// literals and fixed operands folded, anything that varies left unknown.
|
// literals and fixed operands folded, anything that varies left unknown.
|
||||||
func constantZeroDivisor(n calcNode, fields map[string]node) (string, bool) {
|
func constantZeroDivisor(n grammar.CalcNode, operand func(string) []node) (string, bool) {
|
||||||
switch n := n.(type) {
|
switch n := n.(type) {
|
||||||
case calcNeg:
|
case grammar.CalcNeg:
|
||||||
return constantZeroDivisor(n.x, fields)
|
return constantZeroDivisor(n.X, operand)
|
||||||
case calcBin:
|
case grammar.CalcBin:
|
||||||
if n.op == '/' {
|
if n.Operator == '/' {
|
||||||
if v, known := constantValue(n.r, fields); known && v == 0 {
|
if v, known := constantValue(n.R, operand); known && v == 0 {
|
||||||
return calcText(n.r), true
|
return grammar.CalcText(n.R), true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if d, zero := constantZeroDivisor(n.l, fields); zero {
|
if d, zero := constantZeroDivisor(n.L, operand); zero {
|
||||||
return d, true
|
return d, true
|
||||||
}
|
}
|
||||||
return constantZeroDivisor(n.r, fields)
|
return constantZeroDivisor(n.R, operand)
|
||||||
}
|
}
|
||||||
return "", false
|
return "", false
|
||||||
}
|
}
|
||||||
|
|
||||||
// constantValue evaluates an expression whose every operand is fixed.
|
// constantValue evaluates an expression whose every operand is fixed.
|
||||||
func constantValue(n calcNode, fields map[string]node) (float64, bool) {
|
func constantValue(n grammar.CalcNode, operand func(string) []node) (float64, bool) {
|
||||||
switch n := n.(type) {
|
switch n := n.(type) {
|
||||||
case calcNum:
|
case grammar.CalcNum:
|
||||||
return float64(n), true
|
return float64(n), true
|
||||||
case calcVar:
|
case grammar.CalcVar:
|
||||||
t, ok := fields[string(n)].(*template)
|
nodes := operand(n.Name)
|
||||||
if !ok || !t.fixed || t.repeat > 1 {
|
if len(nodes) != 1 {
|
||||||
return 0, false
|
return 0, false
|
||||||
}
|
}
|
||||||
v, err := strconv.ParseFloat(strings.TrimSpace(t.lit), 64)
|
t, ok := nodes[0].(*template)
|
||||||
|
if !ok || t.repeat != 1 {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
lit, fixed := t.fixedText()
|
||||||
|
if !fixed {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
v, err := strconv.ParseFloat(strings.TrimSpace(lit), 64)
|
||||||
return v, err == nil
|
return v, err == nil
|
||||||
case calcNeg:
|
case grammar.CalcNeg:
|
||||||
v, ok := constantValue(n.x, fields)
|
v, ok := constantValue(n.X, operand)
|
||||||
return -v, ok
|
return -v, ok
|
||||||
case calcBin:
|
case grammar.CalcBin:
|
||||||
l, lok := constantValue(n.l, fields)
|
l, lok := constantValue(n.L, operand)
|
||||||
r, rok := constantValue(n.r, fields)
|
r, rok := constantValue(n.R, operand)
|
||||||
if !lok || !rok {
|
if !lok || !rok {
|
||||||
return 0, false
|
return 0, false
|
||||||
}
|
}
|
||||||
return calcBin{n.op, calcNum(l), calcNum(r)}.eval(nil), true
|
return builtinfunc.Arith(n.Operator, l, r), true
|
||||||
}
|
}
|
||||||
return 0, false
|
return 0, false
|
||||||
}
|
}
|
||||||
|
|
||||||
// calcText spells an expression node the way an author would read it.
|
// neverNumeric reports a node no render of which is a number: a null, a repeat of 0, fixed
|
||||||
func calcText(n calcNode) string {
|
// text that does not parse, or a choice of only such items. text is one such render.
|
||||||
switch n := n.(type) {
|
|
||||||
case calcNum:
|
|
||||||
return strconv.FormatFloat(float64(n), 'f', -1, 64)
|
|
||||||
case calcVar:
|
|
||||||
return string(n)
|
|
||||||
case calcNeg:
|
|
||||||
return "-" + calcText(n.x)
|
|
||||||
case calcBin:
|
|
||||||
return "(" + calcText(n.l) + " " + string(n.op) + " " + calcText(n.r) + ")"
|
|
||||||
}
|
|
||||||
return "?"
|
|
||||||
}
|
|
||||||
|
|
||||||
// neverNumeric reports a node no render of which is a number: a null, fixed text that
|
|
||||||
// does not parse, or a choice of only such items. text is one such render.
|
|
||||||
func neverNumeric(n node) (text string, never bool) {
|
func neverNumeric(n node) (text string, never bool) {
|
||||||
switch n := n.(type) {
|
switch n := n.(type) {
|
||||||
case *null:
|
case *nullItem:
|
||||||
return "", true
|
return "", true
|
||||||
case *template:
|
case *template:
|
||||||
if !n.fixed || n.repeat > 1 {
|
lit, fixed := n.fixedText()
|
||||||
|
if n.repeat == 0 {
|
||||||
|
return "", true
|
||||||
|
}
|
||||||
|
if !fixed || n.repeat != 1 {
|
||||||
return "", false
|
return "", false
|
||||||
}
|
}
|
||||||
if _, err := strconv.ParseFloat(strings.TrimSpace(n.lit), 64); err != nil {
|
if _, err := strconv.ParseFloat(strings.TrimSpace(lit), 64); err != nil {
|
||||||
return n.lit, true
|
return lit, true
|
||||||
}
|
}
|
||||||
case *choice:
|
case *choice:
|
||||||
for _, it := range n.items {
|
return allNeverNumeric(n.items)
|
||||||
t, itemNever := neverNumeric(it)
|
|
||||||
if !itemNever {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
text = t
|
|
||||||
}
|
|
||||||
return text, true
|
|
||||||
}
|
}
|
||||||
return "", false
|
return "", false
|
||||||
}
|
}
|
||||||
|
|
||||||
// calcPrep parses the expression and decimals once, at compile time, and places each
|
// allNeverNumeric reports whether no render of any of nodes is a number; false for no nodes.
|
||||||
// operand name at the position expand will read it into. checkCalc proved both args
|
func allNeverNumeric(nodes []node) (text string, never bool) {
|
||||||
// valid, so no step here can fail; dp -1 prints the minimal form.
|
for _, n := range nodes {
|
||||||
func calcPrep(args []string) callFn {
|
t, nodeNever := neverNumeric(n)
|
||||||
expr, err := parseCalc(args[0])
|
if !nodeNever {
|
||||||
if err != nil { // a nil AST would be a nil dereference per render, with no message
|
return "", false
|
||||||
panic(fmt.Sprintf("fejkdata: calc(%q) reached prep unparsed: %v", args[0], err))
|
|
||||||
}
|
|
||||||
at := make(map[string]int)
|
|
||||||
for i, name := range calcVars(expr) {
|
|
||||||
at[name] = i
|
|
||||||
}
|
|
||||||
placed := indexVars(expr, at)
|
|
||||||
dp := calcDecimals(args)
|
|
||||||
return func(_ *session, _ string, operands []string) string {
|
|
||||||
return formatFloat(placed.eval(operands), dp)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// calcDecimals is a calc's decimals count, or -1 for the shortest form.
|
|
||||||
func calcDecimals(args []string) int {
|
|
||||||
if len(args) == 2 {
|
|
||||||
return atoi(args[1])
|
|
||||||
}
|
|
||||||
return -1
|
|
||||||
}
|
|
||||||
|
|
||||||
// indexVars replaces each operand name with its position in the values expand reads.
|
|
||||||
// Both sides take that order from calcVars, so they cannot drift.
|
|
||||||
func indexVars(n calcNode, at map[string]int) calcNode {
|
|
||||||
switch n := n.(type) {
|
|
||||||
case calcVar:
|
|
||||||
i, placed := at[string(n)]
|
|
||||||
if !placed { // calcVars named every operand, so a miss means the two disagree
|
|
||||||
panic(fmt.Sprintf("fejkdata: calc operand %q is not among the names read for it", string(n)))
|
|
||||||
}
|
}
|
||||||
return calcIdx(i)
|
text = t
|
||||||
case calcNeg:
|
|
||||||
return calcNeg{indexVars(n.x, at)}
|
|
||||||
case calcBin:
|
|
||||||
return calcBin{n.op, indexVars(n.l, at), indexVars(n.r, at)}
|
|
||||||
}
|
}
|
||||||
return n
|
return text, len(nodes) > 0
|
||||||
}
|
|
||||||
|
|
||||||
// calcOperands lists the sibling-field names a calc's args read. checkCalc reports
|
|
||||||
// an expression that does not parse, so one that does not simply names nothing.
|
|
||||||
func calcOperands(args []string) []string {
|
|
||||||
if len(args) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
expr, err := parseCalc(args[0])
|
|
||||||
if err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return calcVars(expr)
|
|
||||||
}
|
|
||||||
|
|
||||||
// calcVars lists the distinct field names an expression reads, in the order it first
|
|
||||||
// names each. That order is the contract between expand, which reads the operands
|
|
||||||
// into a slice, and indexVars, which places each name at its position in it.
|
|
||||||
func calcVars(n calcNode) []string {
|
|
||||||
var out []string
|
|
||||||
seen := map[string]bool{}
|
|
||||||
var walk func(calcNode)
|
|
||||||
walk = func(n calcNode) {
|
|
||||||
switch n := n.(type) {
|
|
||||||
case calcVar:
|
|
||||||
if name := string(n); !seen[name] {
|
|
||||||
seen[name] = true
|
|
||||||
out = append(out, name)
|
|
||||||
}
|
|
||||||
case calcNeg:
|
|
||||||
walk(n.x)
|
|
||||||
case calcBin:
|
|
||||||
walk(n.l)
|
|
||||||
walk(n.r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
walk(n)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// calcParser is a recursive-descent parser over the expression runes, threading
|
|
||||||
// expr -> term -> factor for the standard * / before + - precedence.
|
|
||||||
type calcParser struct {
|
|
||||||
rs []rune
|
|
||||||
pos int
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseCalc parses a whole expression, requiring it to consume all input.
|
|
||||||
func parseCalc(expr string) (calcNode, error) {
|
|
||||||
p := &calcParser{rs: []rune(expr)}
|
|
||||||
if p.space(); p.pos >= len(p.rs) {
|
|
||||||
return nil, fmt.Errorf("empty expression")
|
|
||||||
}
|
|
||||||
n, err := p.expr()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if p.space(); p.pos != len(p.rs) {
|
|
||||||
return nil, fmt.Errorf("unexpected %q", string(p.rs[p.pos:]))
|
|
||||||
}
|
|
||||||
return n, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *calcParser) space() {
|
|
||||||
for p.pos < len(p.rs) && unicode.IsSpace(p.rs[p.pos]) {
|
|
||||||
p.pos++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *calcParser) expr() (calcNode, error) { return p.binary(p.term, '+', '-') }
|
|
||||||
func (p *calcParser) term() (calcNode, error) { return p.binary(p.factor, '*', '/') }
|
|
||||||
|
|
||||||
// binary parses a left-associative run of next() operands joined by the given
|
|
||||||
// operators, the one shape expr and term share.
|
|
||||||
func (p *calcParser) binary(next func() (calcNode, error), ops ...byte) (calcNode, error) {
|
|
||||||
n, err := next()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for {
|
|
||||||
p.space()
|
|
||||||
if p.pos >= len(p.rs) || !contains(ops, byte(p.rs[p.pos])) {
|
|
||||||
return n, nil
|
|
||||||
}
|
|
||||||
op := byte(p.rs[p.pos])
|
|
||||||
p.pos++
|
|
||||||
r, err := next()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
n = calcBin{op, n, r}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// factor is a table-shaped scanner, one case per token kind, kept whole on purpose.
|
|
||||||
func (p *calcParser) factor() (calcNode, error) {
|
|
||||||
p.space()
|
|
||||||
if p.pos >= len(p.rs) {
|
|
||||||
return nil, fmt.Errorf("unexpected end of expression")
|
|
||||||
}
|
|
||||||
switch c := p.rs[p.pos]; {
|
|
||||||
case c == '-':
|
|
||||||
p.pos++
|
|
||||||
x, err := p.factor()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return calcNeg{x}, nil
|
|
||||||
case c == '(':
|
|
||||||
p.pos++
|
|
||||||
n, err := p.expr()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if p.space(); p.pos >= len(p.rs) || p.rs[p.pos] != ')' {
|
|
||||||
return nil, fmt.Errorf("missing ')'")
|
|
||||||
}
|
|
||||||
p.pos++
|
|
||||||
return n, nil
|
|
||||||
case c == '.' || c >= '0' && c <= '9':
|
|
||||||
return p.number()
|
|
||||||
case c == '_' || unicode.IsLetter(c):
|
|
||||||
return p.ident()
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("unexpected %q", string(c))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *calcParser) number() (calcNode, error) {
|
|
||||||
start, dot := p.pos, false
|
|
||||||
for p.pos < len(p.rs) {
|
|
||||||
if c := p.rs[p.pos]; c >= '0' && c <= '9' {
|
|
||||||
p.pos++
|
|
||||||
} else if c == '.' && !dot {
|
|
||||||
dot, p.pos = true, p.pos+1
|
|
||||||
} else {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
v, err := strconv.ParseFloat(string(p.rs[start:p.pos]), 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("bad number %q", string(p.rs[start:p.pos]))
|
|
||||||
}
|
|
||||||
return calcNum(v), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ident reads a field name: a letter or '_', then letters, digits or '_'. A '-'
|
|
||||||
// is always the minus operator, so a hyphenated field name can't be an operand.
|
|
||||||
func (p *calcParser) ident() (calcNode, error) {
|
|
||||||
start := p.pos
|
|
||||||
for p.pos < len(p.rs) {
|
|
||||||
if c := p.rs[p.pos]; c == '_' || unicode.IsLetter(c) || unicode.IsDigit(c) {
|
|
||||||
p.pos++
|
|
||||||
} else {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return calcVar(string(p.rs[start:p.pos])), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func contains(bs []byte, b byte) bool {
|
|
||||||
for _, x := range bs {
|
|
||||||
if x == b {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
|
|||||||
+140
-64
@@ -3,8 +3,11 @@ package fejkdata
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
"math"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata/internal/grammar"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestCalcArithmetic pins the operators, precedence, parentheses and unary minus
|
// TestCalcArithmetic pins the operators, precedence, parentheses and unary minus
|
||||||
@@ -72,6 +75,9 @@ func TestCalcFields(t *testing.T) {
|
|||||||
if got := mustRender(t, engine(1), `{"format":"{calc(a - b)}","a":{"format":"{n}","n":"10"},"b":"3"}`); got != "7" {
|
if got := mustRender(t, engine(1), `{"format":"{calc(a - b)}","a":{"format":"{n}","n":"10"},"b":"3"}`); got != "7" {
|
||||||
t.Fatalf("calc(a - b) = %q, want 7", got)
|
t.Fatalf("calc(a - b) = %q, want 7", got)
|
||||||
}
|
}
|
||||||
|
if got := mustRender(t, engine(1), `{"format":"{calc(b * (a - b))}","a":"10","b":"3"}`); got != "21" {
|
||||||
|
t.Fatalf("calc(b * (a - b)) = %q, want 21", got)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestCalcNonNumericIsNaN pins the never-fail rule: a field that sometimes does
|
// TestCalcNonNumericIsNaN pins the never-fail rule: a field that sometimes does
|
||||||
@@ -94,43 +100,26 @@ func TestCalcReproducible(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestTokenOperandsReadsOnlyAnOperandBuiltin pins what the helper answers for a body that
|
// TestParsedCallReadsOnlyAnOperandBuiltin pins the operands a parsed call carries for a
|
||||||
// is not a calc, and for one whose expression does not parse: nothing, either way.
|
// call that is not a calc, and for one whose expression does not parse: none, either way.
|
||||||
// checkCalc is what reports a bad expression, so the callers that run after it
|
func TestParsedCallReadsOnlyAnOperandBuiltin(t *testing.T) {
|
||||||
// never meet one — but they must not have to depend on that order to be safe.
|
for _, format := range []string{"{luhn()}", "{calc()}", "{calc(1 +)}", "{calc(()}"} {
|
||||||
func TestTokenOperandsReadsOnlyAnOperandBuiltin(t *testing.T) {
|
if toks, err := grammar.ParseFormat(format); err != nil || len(toks) != 1 || tokenReads(toks[0]) != nil {
|
||||||
for _, body := range []string{"plain", "luhn()", "calc()", "calc(1 +)", "calc(()"} {
|
t.Errorf("ParseFormat(%q) = %+v, %v, want one call reading no operand", format, toks, err)
|
||||||
if got := tokenOperands(body); got != nil {
|
|
||||||
t.Errorf("tokenOperands(%q) = %v, want none", body, got)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if got := tokenOperands("calc(net * qty)"); len(got) != 2 {
|
if toks, err := grammar.ParseFormat("{calc(net * qty)}"); err != nil || len(toks) != 1 || len(tokenReads(toks[0])) != 2 {
|
||||||
t.Errorf("tokenOperands(calc(net * qty)) = %v, want both operands", got)
|
t.Errorf("ParseFormat({calc(net * qty)}) = %+v, %v, want both operands", toks, err)
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCalcGuardsPanic pins the guards on calc's own invariants: checkCalc parsed
|
|
||||||
// the expression before prep sees it, and indexVars places every name calcVars
|
|
||||||
// read, so a break in either is reported rather than computed around.
|
|
||||||
func TestCalcGuardsPanic(t *testing.T) {
|
|
||||||
for name, call := range map[string]func(){
|
|
||||||
"prep on an expression that does not parse": func() { calcPrep([]string{"1 +"}) },
|
|
||||||
"an operand name never placed": func() { calcVar("n").eval(nil) },
|
|
||||||
"indexVars on a name it did not read": func() { indexVars(calcVar("n"), map[string]int{}) },
|
|
||||||
} {
|
|
||||||
mustPanic(t, name, call)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- one draw, one value: a calc operand reads the expansion's draw ---
|
// --- one draw, one value: a calc operand reads the expansion's draw ---
|
||||||
|
|
||||||
// TestCalcOperandReadsTheExpansionsDraw pins the one-draw rule for a calc operand.
|
// TestANamedCalcOperandIsTheOneShown pins how a calc computes from the value its format
|
||||||
// A field the format renders and a calc reads is drawn once per expansion, so the
|
// shows: the format and the calc read one name.
|
||||||
// operand shown is the operand computed — the correlation the dotted-path rule
|
func TestANamedCalcOperandIsTheOneShown(t *testing.T) {
|
||||||
// already gives a level (see bound_test.go), applied to a plain sibling.
|
|
||||||
func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) {
|
|
||||||
dir := writeData(t, map[string]string{
|
dir := writeData(t, map[string]string{
|
||||||
"inv": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
|
"inv": `{"format":"{net as n}{qty as q}{n} x {q} = {calc(n * q, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
|
||||||
})
|
})
|
||||||
f := newGenerator(t, dir, WithSeed(3))
|
f := newGenerator(t, dir, WithSeed(3))
|
||||||
for i := 0; i < 300; i++ {
|
for i := 0; i < 300; i++ {
|
||||||
@@ -145,39 +134,23 @@ func TestCalcOperandReadsTheExpansionsDraw(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestCalcOperandSharesOneDraw pins the reach of that hold: the draw belongs to the
|
// TestACalcOperandDrawsAfresh pins that a calc reading a field draws it as any {…} does.
|
||||||
// expansion, not to the calc, so the bare token rendering the same name reads it too.
|
func TestACalcOperandDrawsAfresh(t *testing.T) {
|
||||||
func TestCalcOperandSharesOneDraw(t *testing.T) {
|
dir := writeData(t, map[string]string{"same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`})
|
||||||
dir := writeData(t, map[string]string{
|
|
||||||
"same": `{"format":"{w} {calc(w)}","w":["1","2","3","4","5"]}`,
|
|
||||||
})
|
|
||||||
f := newGenerator(t, dir, WithSeed(5))
|
f := newGenerator(t, dir, WithSeed(5))
|
||||||
for i := 0; i < 200; i++ {
|
for i := 0; i < 200; i++ {
|
||||||
got := fake(t, f, "same")
|
if p := strings.Fields(fake(t, f, "same")); p[0] != p[1] {
|
||||||
if p := strings.Fields(got); len(p) != 2 || p[0] != p[1] {
|
|
||||||
t.Fatalf("same = %q, want one value twice", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFieldNoCalcReadsDrawsEachTime guards the boundary: only a name a calc reads is
|
|
||||||
// held, so an ordinary {w} {w} still draws twice.
|
|
||||||
func TestFieldNoCalcReadsDrawsEachTime(t *testing.T) {
|
|
||||||
dir := writeData(t, map[string]string{"two": `{"format":"{w} {w}","w":["1","2","3","4","5"]}`})
|
|
||||||
f := newGenerator(t, dir, WithSeed(5))
|
|
||||||
for i := 0; i < 200; i++ {
|
|
||||||
if p := strings.Fields(fake(t, f, "two")); p[0] != p[1] {
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
t.Fatal("{w} {w} never differed in 200 draws, want two independent draws")
|
t.Fatal("{w} {calc(w)} never differed in 200 draws, want two independent draws")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestCalcHoldIsPerExpansion pins the scope of the hold: each repeat iteration is
|
// TestANamedCalcOperandPicksEachIteration pins that a name a repeat binds picks again on each
|
||||||
// its own expansion, so it draws again while staying self-consistent.
|
// iteration, staying one value within it.
|
||||||
func TestCalcHoldIsPerExpansion(t *testing.T) {
|
func TestANamedCalcOperandPicksEachIteration(t *testing.T) {
|
||||||
dir := writeData(t, map[string]string{
|
dir := writeData(t, map[string]string{
|
||||||
"rep": `{"format":"{n}={calc(n * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`,
|
"rep": `{"format":"{n as m}{m}={calc(m * 1)}","repeat":8,"separator":" ","n":["2","3","4","5","6","7","8","9"]}`,
|
||||||
})
|
})
|
||||||
f := newGenerator(t, dir, WithSeed(11))
|
f := newGenerator(t, dir, WithSeed(11))
|
||||||
varied := false
|
varied := false
|
||||||
@@ -198,12 +171,12 @@ func TestCalcHoldIsPerExpansion(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestCalcHoldIsPerTemplate pins that a nested template holds its own: the inner
|
// TestTwoNamesPickApart pins that a nested template's name is a pick of its own: the inner
|
||||||
// {v} and its calc agree with each other, not with the outer pair.
|
// pair agrees with itself, not with the outer pair.
|
||||||
func TestCalcHoldIsPerTemplate(t *testing.T) {
|
func TestTwoNamesPickApart(t *testing.T) {
|
||||||
dir := writeData(t, map[string]string{
|
dir := writeData(t, map[string]string{
|
||||||
"nest": `{"format":"{v}={calc(v * 1)} {inner}","v":["2","3","4","5","6","7","8","9"],
|
"nest": `{"format":"{v as a}{a}={calc(a * 1)} {inner}","v":["2","3","4","5","6","7","8","9"],
|
||||||
"inner":{"format":"{v}={calc(v * 1)}","v":["2","3","4","5","6","7","8","9"]}}`,
|
"inner":{"format":"{v as b}{b}={calc(b * 1)}","v":["2","3","4","5","6","7","8","9"]}}`,
|
||||||
})
|
})
|
||||||
f := newGenerator(t, dir, WithSeed(13))
|
f := newGenerator(t, dir, WithSeed(13))
|
||||||
differed := false
|
differed := false
|
||||||
@@ -231,7 +204,7 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
|
|||||||
`{"format":"{calc(x * 2)}","x":["a","b"]}`: `"x"`,
|
`{"format":"{calc(x * 2)}","x":["a","b"]}`: `"x"`,
|
||||||
`{"format":"{calc(x + y)}","x":"1","y":"{{2}}"}`: `"{2}"`,
|
`{"format":"{calc(x + y)}","x":"1","y":"{{2}}"}`: `"{2}"`,
|
||||||
} {
|
} {
|
||||||
_, err := compile(parse(t, src))
|
_, err := resolved(t, src)
|
||||||
if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "never a number") {
|
if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "never a number") {
|
||||||
t.Errorf("compile(%s) = %v, want the operand rejected naming %s", src, err, want)
|
t.Errorf("compile(%s) = %v, want the operand rejected naming %s", src, err, want)
|
||||||
}
|
}
|
||||||
@@ -242,7 +215,7 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
|
|||||||
`{"format":"{calc(x * 2)}","x":["1","abc"]}`,
|
`{"format":"{calc(x * 2)}","x":["1","abc"]}`,
|
||||||
`{"format":"{calc(x * 2)}","x":"{digits(2)}"}`,
|
`{"format":"{calc(x * 2)}","x":"{digits(2)}"}`,
|
||||||
} {
|
} {
|
||||||
if _, err := compile(parse(t, ok)); err != nil {
|
if _, err := resolved(t, ok); err != nil {
|
||||||
t.Errorf("compile(%s) = %v, want it accepted", ok, err)
|
t.Errorf("compile(%s) = %v, want it accepted", ok, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -250,14 +223,14 @@ func TestCalcOverANeverNumericOperandIsRejected(t *testing.T) {
|
|||||||
|
|
||||||
func TestCalcConstantZeroDivisorIsRejected(t *testing.T) {
|
func TestCalcConstantZeroDivisorIsRejected(t *testing.T) {
|
||||||
for src, want := range map[string]string{
|
for src, want := range map[string]string{
|
||||||
`"{calc(1/0)}"`: "divides by 0",
|
`"{calc(1/0)}"`: "divides by 0",
|
||||||
`"{calc(2/(1-1))}"`: "divides by (1 - 1)",
|
`"{calc(2/(1-1))}"`: "divides by (1 - 1)",
|
||||||
`{"format":"{calc(x/y)}","x":"1","y":"0"}`: "divides by y",
|
`{"format":"{calc(x/y)}","x":"1","y":"0"}`: "divides by y",
|
||||||
`{"format":"{calc(x/(y*2))}","x":"1","y":" 0 "}`: "divides by (y * 2)",
|
`{"format":"{calc(x/(y*2))}","x":"1","y":" 0 "}`: "divides by (y * 2)",
|
||||||
`{"format":"{calc((a/0)+b)}","a":"1","b":"2"}`: "divides by 0",
|
`{"format":"{calc((a/0)+b)}","a":"1","b":"2"}`: "divides by 0",
|
||||||
`{"format":"{calc(a/-0)}","a":"1"}`: "divides by -0",
|
`{"format":"{calc(a/-0)}","a":"1"}`: "divides by -0",
|
||||||
} {
|
} {
|
||||||
if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) {
|
if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
|
||||||
t.Errorf("compile(%s) = %v, want the constant zero divisor rejected naming %q", src, err, want)
|
t.Errorf("compile(%s) = %v, want the constant zero divisor rejected naming %q", src, err, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -272,3 +245,106 @@ func TestCalcConstantZeroDivisorIsRejected(t *testing.T) {
|
|||||||
}
|
}
|
||||||
t.Fatal("a sometimes-zero divisor never printed +Inf in 50 draws")
|
t.Fatal("a sometimes-zero divisor never printed +Inf in 50 draws")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCalcCompileErrors(t *testing.T) {
|
||||||
|
for _, bad := range []string{
|
||||||
|
`"{calc(1/0)}"`, // a constant zero divisor
|
||||||
|
`{"format":"{calc(x/y)}","x":"1","y":"0"}`, // a fixed zero divisor
|
||||||
|
`"{calc()}"`, // calc needs an expression
|
||||||
|
`"{calc(1 +)}"`, // dangling operator
|
||||||
|
`"{calc((1 + 2)}"`, // unbalanced parenthesis
|
||||||
|
`"{calc(1 2)}"`, // two operands, no operator
|
||||||
|
`"{calc(price)}"`, // operand names no field
|
||||||
|
`"{calc(1, 2, 3)}"`, // too many args
|
||||||
|
`"{calc(1, x)}"`, // decimals arg not an integer
|
||||||
|
`"{calc(1, -1)}"`, // decimals negative
|
||||||
|
} {
|
||||||
|
if _, err := resolved(t, bad); err == nil {
|
||||||
|
t.Errorf("compile(%s) = nil error, want error", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCalcReadsAName(t *testing.T) {
|
||||||
|
dir := writeData(t, map[string]string{
|
||||||
|
"n": `["2","3","5","7"]`,
|
||||||
|
"card": `"{/n as k}{k} x 2 = {calc(k * 2)}"`,
|
||||||
|
})
|
||||||
|
f := newGenerator(t, dir, WithSeed(1))
|
||||||
|
for i := 0; i < 50; i++ {
|
||||||
|
var k, double int
|
||||||
|
got := fake(t, f, "card")
|
||||||
|
if _, err := fmt.Sscanf(got, "%d x 2 = %d", &k, &double); err != nil || double != k*2 {
|
||||||
|
t.Fatalf("card = %q, want the calc to compute from the pick {k} prints", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCalcReadsANameItReadsNowhereElse(t *testing.T) {
|
||||||
|
dir := writeData(t, map[string]string{
|
||||||
|
"n": `["2","3"]`,
|
||||||
|
"card": `"{/n as k}{calc(k * 2)}"`,
|
||||||
|
})
|
||||||
|
f := newGenerator(t, dir, WithSeed(1))
|
||||||
|
if got := fake(t, f, "card"); got != "4" && got != "6" {
|
||||||
|
t.Fatalf("card = %q, want 4 or 6", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCalcRefusesABadOperand(t *testing.T) {
|
||||||
|
for card, want := range map[string]string{
|
||||||
|
`"{/word as w}{w}{calc(w * 2)}"`: `operand "w" is never a number: it renders "def"`,
|
||||||
|
`"{/zero as z}{z}{calc(1 / z)}"`: `divides by z, which is always zero`,
|
||||||
|
`"{calc(nope * 2)}"`: `no field "nope"`,
|
||||||
|
`"{/word as w}{w}{calc(nope * 2)}"`: `no field or name "nope"; the names bound here are "w"`,
|
||||||
|
} {
|
||||||
|
dir := writeData(t, map[string]string{
|
||||||
|
"word": `["abc","def"]`,
|
||||||
|
"zero": `"0"`,
|
||||||
|
"card": card,
|
||||||
|
})
|
||||||
|
if _, err := New(withShipped(), WithDataPath(dir), WithSeed(1)); err == nil || !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("New with card %s = %v, want an error containing %q", card, err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestATypedColumnRefusesACalcOverNoDigits(t *testing.T) {
|
||||||
|
_, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{
|
||||||
|
"row": `{"format":"","x":{"format":"{calc(d + 1)}","d":"{digits(0)}","datatype":"integer"}}`,
|
||||||
|
})))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "{digits(0)} prints nothing, which is no number") {
|
||||||
|
t.Fatalf("New = %v, want a calc over {digits(0)} refused in a typed column", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestATypedColumnProvesACalcOverAName(t *testing.T) {
|
||||||
|
dir := writeData(t, map[string]string{
|
||||||
|
"n": `"{int(1,9)}"`,
|
||||||
|
"order": `{"format":"{/n as k}{k}","double":{"format":"{calc(k * 2)}","datatype":"integer"}}`,
|
||||||
|
})
|
||||||
|
f := newGenerator(t, dir, WithSeed(1))
|
||||||
|
r, err := f.FakeRecord("order")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if v, err := strconv.Atoi(r.Columns()[0].Value); err != nil || v < 2 || v > 18 || v%2 != 0 {
|
||||||
|
t.Fatalf("double = %q, want an even integer in 2..18", r.Columns()[0].Value)
|
||||||
|
}
|
||||||
|
if !strings.Contains(r.JSON(), `"double":`+r.Columns()[0].Value) {
|
||||||
|
t.Fatalf("JSON = %s, want double written as a number", r.JSON())
|
||||||
|
}
|
||||||
|
bad := writeData(t, map[string]string{
|
||||||
|
"word": `["1","x"]`,
|
||||||
|
"order": `{"format":"{/word as w}{w}","double":{"format":"{calc(w * 2)}","datatype":"integer"}}`,
|
||||||
|
})
|
||||||
|
if _, err := New(withShipped(), WithDataPath(bad), WithSeed(1)); err == nil || !strings.Contains(err.Error(), `operand "w": "x" is not a number`) {
|
||||||
|
t.Fatalf("New = %v, want the typed column refused over an operand that is not always a number", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARepeatOfZeroIsNeverANumber(t *testing.T) {
|
||||||
|
if _, err := resolved(t, `{"format":"{calc(x / y)}","x":"1","y":{"format":"0","repeat":0}}`); err == nil || !strings.Contains(err.Error(), `operand "y" is never a number: it renders ""`) {
|
||||||
|
t.Errorf("a calc over a repeat of 0 = %v, want the operand refused as never a number", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+706
@@ -0,0 +1,706 @@
|
|||||||
|
// Package cli is the fejkdata command: it renders the template on its stdin with fake
|
||||||
|
// values from the modules it is given and any directories layered over them. A program
|
||||||
|
// passing its own modules to Main is a fejkdata CLI carrying them, for example
|
||||||
|
// os.Exit(cli.Main(append(data.Modules(), mydata)...)).
|
||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"math/rand/v2"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"runtime/debug"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata"
|
||||||
|
)
|
||||||
|
|
||||||
|
const usage = `Usage: <template> | COMMAND [flags]
|
||||||
|
COMMAND --list | --help | --version
|
||||||
|
|
||||||
|
COMMAND reads one template from stdin and renders it:
|
||||||
|
|
||||||
|
echo 'id: {int(1,999)}' | COMMAND
|
||||||
|
echo '{"format":"{x}","x":["bosse","lina"]}' | COMMAND
|
||||||
|
COMMAND < template.txt
|
||||||
|
|
||||||
|
COMMAND <<'EOF'
|
||||||
|
born {date(1990-01-01,2010-12-31,'2006-01-02')}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
A template is a format string, or a JSON object, array, string or null. Its {…} tokens
|
||||||
|
reach the data by reference from the root: {/path}, for any path --list prints, shipped
|
||||||
|
or layered with --data-path. A reference names a category or a dotted path into one, a
|
||||||
|
table's row by key or name ({/table[KEY]}), and .. steps up from a row to the row it
|
||||||
|
links to.
|
||||||
|
|
||||||
|
Under --format text, the default, what you send is what renders, and COMMAND adds
|
||||||
|
nothing: a format string keeps every byte, so echo's newline ends each render, and
|
||||||
|
printf '%s' sends none. -n joins renders with --separator, empty by default. A JSON
|
||||||
|
object, array or string drops the whitespace around it, and null alone renders nothing;
|
||||||
|
42 or true alone is a format string, printed as written. An empty or blank stdin is a
|
||||||
|
format string like any other, and renders as sent. A quoted heredoc, <<'EOF', passes $,
|
||||||
|
a backslash or a quote as written.
|
||||||
|
|
||||||
|
With --format json, ndjson, csv or sql the template must be a record — a JSON
|
||||||
|
template whose fields are its columns, or a lone reference such as {/path}:
|
||||||
|
one reference and nothing else, with one newline after it allowed — and the rows are
|
||||||
|
written as one JSON array, one JSON object per line, one CSV row (after a header), or
|
||||||
|
one INSERT.
|
||||||
|
|
||||||
|
-d, --data-path D a data directory to layer over the shipped data (repeatable; last wins on a clash)
|
||||||
|
--format F output form: text (default), json, ndjson, csv or sql
|
||||||
|
-h, --help print this help, then exit
|
||||||
|
--list list the paths the data offers, each written after {/ in a template, then exit;
|
||||||
|
--repeat, --separator, --format and --table do nothing here
|
||||||
|
--no-shipped-data load only the --data-path directories
|
||||||
|
-n, --repeat N render the template N times, 0..1048576 (default 1); 0 prints nothing,
|
||||||
|
[] under --format json, or the header under --format csv
|
||||||
|
-s, --seed N same seed, version and data: identical output (default: a random seed)
|
||||||
|
--separator S string between repeated renders (default empty); refused with a record
|
||||||
|
--format, which has its own separators
|
||||||
|
--table T the INSERT target for --format sql, ignored under any other
|
||||||
|
(default: a lone reference's last segment, else records)
|
||||||
|
--version print the version, then exit
|
||||||
|
|
||||||
|
A short flag's value attaches or follows (-n3, -n 3); short flags bundle (-hn 3);
|
||||||
|
-- ends the flags.
|
||||||
|
|
||||||
|
Exit codes: 0 success, 1 runtime error, 2 misuse.
|
||||||
|
`
|
||||||
|
|
||||||
|
type invocation struct {
|
||||||
|
args []string
|
||||||
|
dirs []string
|
||||||
|
format string
|
||||||
|
help bool
|
||||||
|
list bool
|
||||||
|
noShipped bool
|
||||||
|
repeat int
|
||||||
|
seed uint64
|
||||||
|
seeded bool
|
||||||
|
separator string
|
||||||
|
separatorSet bool
|
||||||
|
table string
|
||||||
|
tableSet bool
|
||||||
|
version bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type flagDef struct {
|
||||||
|
long string
|
||||||
|
short string
|
||||||
|
value bool
|
||||||
|
set func(*invocation, string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
var flagDefs = []flagDef{
|
||||||
|
{"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }},
|
||||||
|
{"format", "", true, func(in *invocation, v string) error { in.format = v; return nil }},
|
||||||
|
{"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }},
|
||||||
|
{"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }},
|
||||||
|
{"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }},
|
||||||
|
{"repeat", "n", true, func(in *invocation, v string) error {
|
||||||
|
n, err := strconv.Atoi(v)
|
||||||
|
if err != nil || n < 0 || n > fejkdata.MaxRepeat {
|
||||||
|
return fmt.Errorf("--repeat needs an integer in 0..%d, got %q", fejkdata.MaxRepeat, v)
|
||||||
|
}
|
||||||
|
in.repeat = n
|
||||||
|
return nil
|
||||||
|
}},
|
||||||
|
{"seed", "s", true, func(in *invocation, v string) error {
|
||||||
|
n, err := strconv.ParseUint(v, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("--seed needs an unsigned integer, got %q", v)
|
||||||
|
}
|
||||||
|
in.seed, in.seeded = n, true
|
||||||
|
return nil
|
||||||
|
}},
|
||||||
|
{"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }},
|
||||||
|
{"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }},
|
||||||
|
{"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }},
|
||||||
|
}
|
||||||
|
|
||||||
|
func flagByLong(name string) *flagDef {
|
||||||
|
for i := range flagDefs {
|
||||||
|
if flagDefs[i].long == name {
|
||||||
|
return &flagDefs[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func flagByShort(name string) *flagDef {
|
||||||
|
for i := range flagDefs {
|
||||||
|
if flagDefs[i].short == name {
|
||||||
|
return &flagDefs[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// flagArg is one flag as written: its definition and the value attached to it,
|
||||||
|
// if any.
|
||||||
|
type flagArg struct {
|
||||||
|
def *flagDef
|
||||||
|
value string
|
||||||
|
inline bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// splitFlags reads one argv element as flags: --name or --name=value, or -abc
|
||||||
|
// where each letter is a short flag and the first that takes a value takes the
|
||||||
|
// rest of the element.
|
||||||
|
func splitFlags(arg string) ([]flagArg, error) {
|
||||||
|
if strings.HasPrefix(arg, "--") {
|
||||||
|
name, value, inline := strings.Cut(arg[2:], "=")
|
||||||
|
def := flagByLong(name)
|
||||||
|
if def == nil {
|
||||||
|
return nil, fmt.Errorf("unknown flag --%s", name)
|
||||||
|
}
|
||||||
|
return []flagArg{{def, value, inline}}, nil
|
||||||
|
}
|
||||||
|
if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil {
|
||||||
|
return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name)
|
||||||
|
}
|
||||||
|
var flags []flagArg
|
||||||
|
letters := arg[1:]
|
||||||
|
for i, r := range letters {
|
||||||
|
letter := string(r)
|
||||||
|
def := flagByShort(letter)
|
||||||
|
if def == nil {
|
||||||
|
return nil, fmt.Errorf("unknown flag -%s", letter)
|
||||||
|
}
|
||||||
|
if !def.value {
|
||||||
|
flags = append(flags, flagArg{def: def})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rest := letters[i+len(letter):]
|
||||||
|
if strings.HasPrefix(rest, "=") {
|
||||||
|
return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s",
|
||||||
|
letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:])
|
||||||
|
}
|
||||||
|
return append(flags, flagArg{def, rest, rest != ""}), nil
|
||||||
|
}
|
||||||
|
return flags, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseArgs(argv []string) (invocation, error) {
|
||||||
|
in := invocation{repeat: 1, format: "text"}
|
||||||
|
for i := 0; i < len(argv); i++ {
|
||||||
|
arg := argv[i]
|
||||||
|
if arg == "--" {
|
||||||
|
in.args = append(in.args, argv[i+1:]...)
|
||||||
|
return in, nil
|
||||||
|
}
|
||||||
|
if len(arg) < 2 || arg[0] != '-' {
|
||||||
|
in.args = append(in.args, arg)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
flags, err := splitFlags(arg)
|
||||||
|
if err != nil {
|
||||||
|
return in, err
|
||||||
|
}
|
||||||
|
for _, fl := range flags {
|
||||||
|
if !fl.def.value {
|
||||||
|
if fl.inline {
|
||||||
|
return in, fmt.Errorf("--%s takes no value", fl.def.long)
|
||||||
|
}
|
||||||
|
_ = fl.def.set(&in, "")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
value := fl.value
|
||||||
|
if !fl.inline {
|
||||||
|
if i++; i >= len(argv) {
|
||||||
|
return in, fmt.Errorf("--%s needs a value", fl.def.long)
|
||||||
|
}
|
||||||
|
value = argv[i]
|
||||||
|
}
|
||||||
|
if err := fl.def.set(&in, value); err != nil {
|
||||||
|
return in, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return in, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordFormat is one way to write a record out: the line each record renders,
|
||||||
|
// the header that precedes the first one, and the open/close frame plus the
|
||||||
|
// between-record separator a document form needs.
|
||||||
|
type recordFormat struct {
|
||||||
|
header func(*fejkdata.Record) string
|
||||||
|
line func(r *fejkdata.Record, table string) string
|
||||||
|
open string
|
||||||
|
close string
|
||||||
|
sep string
|
||||||
|
writes string // what it writes between values, which --separator would seem to set
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordFormats is every --format that writes records. json frames the records
|
||||||
|
// as one array document; ndjson writes one object per line.
|
||||||
|
var recordFormats = map[string]recordFormat{
|
||||||
|
"csv": {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }, sep: "\n", writes: `always separates fields with ","`},
|
||||||
|
"json": {line: jsonLine, open: "[", close: "]", sep: ",\n", writes: "writes the records as one JSON array"},
|
||||||
|
"ndjson": {line: jsonLine, sep: "\n", writes: "writes one record per line"},
|
||||||
|
"sql": {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }, sep: "\n", writes: "writes one INSERT per line"},
|
||||||
|
}
|
||||||
|
|
||||||
|
func jsonLine(r *fejkdata.Record, _ string) string { return r.JSON() }
|
||||||
|
|
||||||
|
func (in invocation) writesRecords() bool {
|
||||||
|
return in.format != "text"
|
||||||
|
}
|
||||||
|
|
||||||
|
// formatNames lists the --format values, text included, for the misuse error.
|
||||||
|
func formatNames() string {
|
||||||
|
names := []string{"text"}
|
||||||
|
for name := range recordFormats {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
return strings.Join(names[:len(names)-1], ", ") + " or " + names[len(names)-1]
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkFlags rejects a flag value or combination that cannot run.
|
||||||
|
func (in invocation) checkFlags() error {
|
||||||
|
if _, ok := recordFormats[in.format]; !ok && in.format != "text" {
|
||||||
|
return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format)
|
||||||
|
}
|
||||||
|
if in.tableSet && in.table == "" && in.format == "sql" {
|
||||||
|
return errors.New("--table names the INSERT target, so it cannot be empty")
|
||||||
|
}
|
||||||
|
if f, records := recordFormats[in.format]; records && in.separatorSet && !in.list {
|
||||||
|
return fmt.Errorf("--separator joins text renders; --format %s %s", in.format, f.writes)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func pipeHint(name string) string { return "echo 'id: {int(1,999)}' | " + name }
|
||||||
|
|
||||||
|
// operandError refuses an argument that is no flag, naming the spelling that reads what
|
||||||
|
// it most likely means: dropping it for -, a file for a file's name, else a template
|
||||||
|
// piped in.
|
||||||
|
func operandError(arg, name string) error {
|
||||||
|
unexpected := "unexpected argument '" + arg + "'"
|
||||||
|
if arg == "-" {
|
||||||
|
return fmt.Errorf("%s; stdin is read already, so drop it", unexpected)
|
||||||
|
}
|
||||||
|
if fi, err := os.Stat(arg); err == nil && fi.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("%s; read a file from stdin: %s < %s", unexpected, name, shellQuoted(arg))
|
||||||
|
}
|
||||||
|
ref := "{/" + strings.TrimPrefix(arg, "/") + "}"
|
||||||
|
_, argLone := loneReference(arg)
|
||||||
|
inline, err := fejkdata.IsTemplate(arg)
|
||||||
|
_, refLone := loneReference(ref)
|
||||||
|
example := pipeHint(name)
|
||||||
|
switch {
|
||||||
|
case argLone || inline:
|
||||||
|
example = piped(arg, name)
|
||||||
|
case err == nil && refLone:
|
||||||
|
example = piped(ref, name)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%s; the template is read from stdin: %s", unexpected, example)
|
||||||
|
}
|
||||||
|
|
||||||
|
// check rejects a flag combination or an operand that cannot run, before any data is
|
||||||
|
// read.
|
||||||
|
func (in invocation) check(name string) error {
|
||||||
|
if err := in.checkFlags(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(in.args) > 0 {
|
||||||
|
return operandError(in.args[0], name)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (in invocation) options(modules []fs.FS, systemSeed uint64) []fejkdata.Option {
|
||||||
|
if in.noShipped {
|
||||||
|
modules = nil
|
||||||
|
}
|
||||||
|
opts := []fejkdata.Option{fejkdata.WithDataFS(modules...)}
|
||||||
|
for _, dir := range in.dirs {
|
||||||
|
opts = append(opts, fejkdata.WithDataPath(dir))
|
||||||
|
}
|
||||||
|
seed := in.seed
|
||||||
|
if !in.seeded {
|
||||||
|
seed = systemSeed
|
||||||
|
}
|
||||||
|
return append(opts, fejkdata.WithSeed(seed))
|
||||||
|
}
|
||||||
|
|
||||||
|
// write streams the input's renders to w, repeat of them joined by the separator; a write
|
||||||
|
// failure surfaces from Flush, bufio keeping the first one.
|
||||||
|
func (in invocation) write(f *fejkdata.Generator, src input, w io.Writer) error {
|
||||||
|
if in.writesRecords() {
|
||||||
|
return in.writeRecords(f, src, w)
|
||||||
|
}
|
||||||
|
t, err := f.NewTemplate(src.template)
|
||||||
|
if err != nil {
|
||||||
|
return templateError(err)
|
||||||
|
}
|
||||||
|
out := bufio.NewWriter(w)
|
||||||
|
for i := 0; i < in.repeat; i++ {
|
||||||
|
if i > 0 {
|
||||||
|
out.WriteString(in.separator)
|
||||||
|
}
|
||||||
|
out.WriteString(t.Fake())
|
||||||
|
}
|
||||||
|
return out.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordStream builds the record drawer for the input, plus the INSERT table
|
||||||
|
// a sql format names.
|
||||||
|
func (in invocation) recordStream(f *fejkdata.Generator, src input) (func() (*fejkdata.Record, error), string, error) {
|
||||||
|
record := func() (*fejkdata.Record, error) { return f.FakeRecord(src.record) }
|
||||||
|
if src.record == "" {
|
||||||
|
t, err := f.NewRecordTemplate(src.template)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", templateError(spacedReference(src.template, err))
|
||||||
|
}
|
||||||
|
record = func() (*fejkdata.Record, error) { return t.Fake(), nil }
|
||||||
|
}
|
||||||
|
table := in.table
|
||||||
|
if table == "" {
|
||||||
|
table = defaultTable(src.record)
|
||||||
|
}
|
||||||
|
return record, table, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeRecords streams a record per line in the chosen format, framing a document
|
||||||
|
// form with its open/close brackets and a header preceding the first record.
|
||||||
|
func (in invocation) writeRecords(f *fejkdata.Generator, src input, w io.Writer) error {
|
||||||
|
record, table, err := in.recordStream(f, src)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
format := recordFormats[in.format]
|
||||||
|
if in.repeat == 0 {
|
||||||
|
return writeNoRecord(record, format, w)
|
||||||
|
}
|
||||||
|
out := bufio.NewWriter(w)
|
||||||
|
if format.open != "" {
|
||||||
|
out.WriteString(format.open)
|
||||||
|
out.WriteByte('\n')
|
||||||
|
}
|
||||||
|
for i := 0; i < in.repeat; i++ {
|
||||||
|
r, err := record()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if i == 0 && format.header != nil {
|
||||||
|
out.WriteString(format.header(r))
|
||||||
|
out.WriteByte('\n')
|
||||||
|
}
|
||||||
|
if i > 0 {
|
||||||
|
out.WriteString(format.sep)
|
||||||
|
}
|
||||||
|
out.WriteString(format.line(r, table))
|
||||||
|
}
|
||||||
|
if format.close != "" {
|
||||||
|
out.WriteByte('\n')
|
||||||
|
out.WriteString(format.close)
|
||||||
|
}
|
||||||
|
out.WriteByte('\n')
|
||||||
|
return out.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeNoRecord writes zero records: the header where the format writes one, an empty document
|
||||||
|
// where it frames one, else nothing. It draws one record, which names the header's columns, so a
|
||||||
|
// lone reference to nothing fails as it would with records.
|
||||||
|
func writeNoRecord(record func() (*fejkdata.Record, error), format recordFormat, w io.Writer) error {
|
||||||
|
r, err := record()
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
return err
|
||||||
|
case format.header != nil:
|
||||||
|
_, err = io.WriteString(w, format.header(r)+"\n")
|
||||||
|
case format.open != "":
|
||||||
|
_, err = io.WriteString(w, format.open+format.close+"\n")
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// defaultTable names the INSERT target when --table is absent: the path's last
|
||||||
|
// segment, or "records" when stdin is not a lone reference.
|
||||||
|
func defaultTable(path string) string {
|
||||||
|
if path == "" {
|
||||||
|
return "records"
|
||||||
|
}
|
||||||
|
segments := strings.Split(withoutSelectors(path), ".")
|
||||||
|
return segments[len(segments)-1]
|
||||||
|
}
|
||||||
|
|
||||||
|
// withoutSelectors is path with its [selectors] cut out.
|
||||||
|
func withoutSelectors(path string) string {
|
||||||
|
var names strings.Builder
|
||||||
|
for depth, i := 0, 0; i < len(path); i++ {
|
||||||
|
switch {
|
||||||
|
case path[i] == '[':
|
||||||
|
depth++
|
||||||
|
case path[i] == ']':
|
||||||
|
depth--
|
||||||
|
case depth == 0:
|
||||||
|
names.WriteByte(path[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// inputError marks a failure that is the input's own fault — nothing piped in, a template
|
||||||
|
// that does not compile, or a stdin that cannot be read. run reports it as misuse (exit 2, with a
|
||||||
|
// pointer to --help), unlike a lone reference to nothing under --format, which is a runtime
|
||||||
|
// error (exit 1).
|
||||||
|
type inputError struct{ error }
|
||||||
|
|
||||||
|
func (e inputError) Unwrap() error { return e.error }
|
||||||
|
|
||||||
|
// jsonSpace is the whitespace JSON allows around a value.
|
||||||
|
const jsonSpace = " \t\r\n"
|
||||||
|
|
||||||
|
// input is what stdin holds: the template, and the path a lone reference names as a
|
||||||
|
// record.
|
||||||
|
type input struct {
|
||||||
|
template string
|
||||||
|
record string
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseInput keeps stdin whole as the template, and names a record where stdin is a lone
|
||||||
|
// reference once JSON's surrounding whitespace, or one final newline, is cut.
|
||||||
|
func parseInput(raw string) input {
|
||||||
|
trimmed := strings.Trim(raw, jsonSpace)
|
||||||
|
text := raw
|
||||||
|
if json.Valid([]byte(trimmed)) {
|
||||||
|
text = trimmed
|
||||||
|
}
|
||||||
|
text, found := strings.CutSuffix(text, "\n")
|
||||||
|
if found {
|
||||||
|
text = strings.TrimSuffix(text, "\r")
|
||||||
|
}
|
||||||
|
record, _ := loneReference(text)
|
||||||
|
return input{template: raw, record: record}
|
||||||
|
}
|
||||||
|
|
||||||
|
// loneReference returns the path that text reads when it is one reference from the root
|
||||||
|
// and nothing else, bare or as a JSON string.
|
||||||
|
func loneReference(text string) (string, bool) {
|
||||||
|
var unquoted string
|
||||||
|
if strings.HasPrefix(text, `"`) && strings.HasSuffix(text, `"`) && json.Unmarshal([]byte(text), &unquoted) == nil {
|
||||||
|
text = unquoted
|
||||||
|
}
|
||||||
|
path, opens := strings.CutPrefix(text, "{/")
|
||||||
|
path, closes := strings.CutSuffix(path, "}")
|
||||||
|
if !opens || !closes || !isPath(path) {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return path, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// isPath reports whether a reference's body after its / is a path alone: no second /, no
|
||||||
|
// folder sigil, no empty segment, and none of the |, ( or " as " a token reads outside a selector.
|
||||||
|
// It copies the grammar, since the CLI reads only the library's public API.
|
||||||
|
func isPath(body string) bool {
|
||||||
|
if body == "" || strings.ContainsAny(body[:1], "/.") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
names := withoutSelectors(body)
|
||||||
|
if strings.ContainsAny(names, "|(") || strings.Contains(names, " as ") || hasEmptySegment(names) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
inline, err := fejkdata.IsTemplate(body)
|
||||||
|
return !inline && err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// hasEmptySegment reports whether a dotted path holds a segment with no name: two dots are a
|
||||||
|
// step up, and only a step up may end the path.
|
||||||
|
func hasEmptySegment(path string) bool {
|
||||||
|
segments := strings.Split(path, ".")
|
||||||
|
stepUp := false
|
||||||
|
for i, seg := range segments[1:] {
|
||||||
|
last := i == len(segments)-2
|
||||||
|
switch {
|
||||||
|
case seg != "":
|
||||||
|
stepUp = false
|
||||||
|
case stepUp != last:
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
stepUp = !stepUp
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// spacedReference adds to a record refusal that whitespace around a lone reference made
|
||||||
|
// the template text.
|
||||||
|
func spacedReference(template string, err error) error {
|
||||||
|
if trimmed := strings.TrimSpace(template); trimmed != template && errors.Is(err, fejkdata.ErrNoColumns) {
|
||||||
|
if path, lone := loneReference(trimmed); lone {
|
||||||
|
return fmt.Errorf("%w; the whitespace around {/%s} makes it text, so remove it to write the record %s", err, path, path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// piped is the command piping template into the command called name; some shells' echo
|
||||||
|
// reads a backslash as an escape, so a template holding one goes through printf.
|
||||||
|
func piped(template, name string) string {
|
||||||
|
if strings.ContainsRune(template, '\\') {
|
||||||
|
return "printf '%s' " + shellQuoted(template) + " | " + name
|
||||||
|
}
|
||||||
|
return "echo " + shellQuoted(template) + " | " + name
|
||||||
|
}
|
||||||
|
|
||||||
|
// shellQuoted is s as a shell reads it back: bare where it holds nothing a shell expands.
|
||||||
|
func shellQuoted(s string) string {
|
||||||
|
if s != "" && !strings.ContainsAny(s, " \t\r\n'\"$`\\*?[]{}()<>|&;#~!") {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Main runs the command on the process's arguments and standard streams over modules, each
|
||||||
|
// passed to [fejkdata.WithDataFS], and returns the exit code: 0 ok, 1 runtime error,
|
||||||
|
// 2 misuse. --no-shipped-data leaves modules out.
|
||||||
|
func Main(modules ...fs.FS) int {
|
||||||
|
var stdin io.Reader = os.Stdin
|
||||||
|
if fi, err := os.Stdin.Stat(); err == nil && fi.Mode()&os.ModeCharDevice != 0 {
|
||||||
|
stdin = nil
|
||||||
|
}
|
||||||
|
return run(commandName(os.Args[0]), modules, os.Args[1:], rand.Uint64(), stdin, os.Stdout, os.Stderr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// commandName is the name the command runs under: the file arg0 names, less a Windows
|
||||||
|
// .exe, or fejkdata where arg0 names none.
|
||||||
|
func commandName(arg0 string) string {
|
||||||
|
name := filepath.Base(arg0)
|
||||||
|
if ext := filepath.Ext(name); strings.EqualFold(ext, ".exe") {
|
||||||
|
name = strings.TrimSuffix(name, ext)
|
||||||
|
}
|
||||||
|
if name == "." || name == "" {
|
||||||
|
return "fejkdata"
|
||||||
|
}
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
|
||||||
|
// run is Main over the given streams, run as the command name, seeded by systemSeed where
|
||||||
|
// no --seed is given. A nil stdin is a terminal, which holds no template.
|
||||||
|
func run(name string, modules []fs.FS, args []string, systemSeed uint64, stdin io.Reader, stdout, stderr io.Writer) int {
|
||||||
|
in, err := parseArgs(args)
|
||||||
|
if err != nil {
|
||||||
|
return misuse(stderr, name, err)
|
||||||
|
}
|
||||||
|
if in.help {
|
||||||
|
io.WriteString(stdout, strings.ReplaceAll(usage, "COMMAND", name))
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if in.version {
|
||||||
|
info, _ := debug.ReadBuildInfo()
|
||||||
|
fmt.Fprintln(stdout, versionLine(name, info))
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if err := in.check(name); err != nil {
|
||||||
|
return misuse(stderr, name, err)
|
||||||
|
}
|
||||||
|
var src input
|
||||||
|
if !in.list {
|
||||||
|
if src, err = readStdin(stdin, name); err != nil {
|
||||||
|
return fail(stderr, name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f, err := fejkdata.New(in.options(modules, systemSeed)...)
|
||||||
|
if err != nil {
|
||||||
|
return fail(stderr, name, err)
|
||||||
|
}
|
||||||
|
if in.list {
|
||||||
|
for _, p := range f.List() {
|
||||||
|
fmt.Fprintln(stdout, p)
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if err := in.write(f, src, stdout); err != nil {
|
||||||
|
return fail(stderr, name, err)
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func readStdin(stdin io.Reader, name string) (input, error) {
|
||||||
|
if stdin == nil {
|
||||||
|
return input{}, inputError{fmt.Errorf("the template comes from stdin, and nothing is piped in: %s; %s --list prints the paths a {/path} reference reads", pipeHint(name), name)}
|
||||||
|
}
|
||||||
|
raw, err := io.ReadAll(stdin)
|
||||||
|
if err != nil {
|
||||||
|
return input{}, inputError{fmt.Errorf("stdin cannot be read: %w", err)}
|
||||||
|
}
|
||||||
|
return parseInput(string(raw)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// templateError is misuse unless the data the template reads failed to load.
|
||||||
|
func templateError(err error) error {
|
||||||
|
if errors.Is(err, fejkdata.ErrLoad) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return inputError{err}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fail(stderr io.Writer, name string, err error) int {
|
||||||
|
var te inputError
|
||||||
|
if errors.As(err, &te) {
|
||||||
|
return misuse(stderr, name, te.error)
|
||||||
|
}
|
||||||
|
report(stderr, name, err)
|
||||||
|
if errors.Is(err, fejkdata.ErrNoColumns) {
|
||||||
|
fmt.Fprintln(stderr, "wrap that JSON in single quotes, which keep a shell from expanding its braces")
|
||||||
|
}
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func misuse(stderr io.Writer, name string, err error) int {
|
||||||
|
report(stderr, name, err)
|
||||||
|
fmt.Fprintf(stderr, "try '%s --help'\n", name)
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
|
||||||
|
// report writes err under the command name, in place of the library's own on each line.
|
||||||
|
func report(stderr io.Writer, name string, err error) {
|
||||||
|
msg := strings.ReplaceAll(strings.TrimPrefix(err.Error(), "fejkdata: "), "\nfejkdata: ", "\n"+name+": ")
|
||||||
|
fmt.Fprintf(stderr, "%s: %s\n", name, msg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// fejkdataModule is the path of the module holding the library; the root package's path is
|
||||||
|
// the module's.
|
||||||
|
var fejkdataModule = reflect.TypeOf((*fejkdata.Generator)(nil)).Elem().PkgPath()
|
||||||
|
|
||||||
|
// versionLine names the fejkdata version the binary is built with. Where another module's
|
||||||
|
// main package carries the CLI, the main module's version comes first, under the command name.
|
||||||
|
func versionLine(name string, info *debug.BuildInfo) string {
|
||||||
|
if info == nil {
|
||||||
|
return "fejkdata devel"
|
||||||
|
}
|
||||||
|
if info.Main.Path == fejkdataModule {
|
||||||
|
return "fejkdata " + moduleVersion(info.Main)
|
||||||
|
}
|
||||||
|
for _, m := range info.Deps {
|
||||||
|
if m.Path == fejkdataModule {
|
||||||
|
return name + " " + moduleVersion(info.Main) + ", fejkdata " + moduleVersion(*m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return name + " " + moduleVersion(info.Main) + ", fejkdata devel"
|
||||||
|
}
|
||||||
|
|
||||||
|
// moduleVersion is the version m is built at: the replacement's where a replace directive
|
||||||
|
// names one, and "devel" where it carries none, as a local directory does.
|
||||||
|
func moduleVersion(m debug.Module) string {
|
||||||
|
if m.Replace != nil {
|
||||||
|
m = *m.Replace
|
||||||
|
}
|
||||||
|
if m.Version == "" || m.Version == "(devel)" {
|
||||||
|
return "devel"
|
||||||
|
}
|
||||||
|
return m.Version
|
||||||
|
}
|
||||||
+961
@@ -0,0 +1,961 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/csv"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"runtime/debug"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
)
|
||||||
|
|
||||||
|
// shipped holds the shipped modules as read from the checkout, each folder under ../data
|
||||||
|
// holding a manifest, so these tests link no data package.
|
||||||
|
var shipped = func() []fs.FS {
|
||||||
|
var modules []fs.FS
|
||||||
|
err := filepath.WalkDir(filepath.Join("..", "data"), func(path string, d fs.DirEntry, err error) error {
|
||||||
|
if err == nil && d.Name() == ".fejkdata.json" {
|
||||||
|
modules = append(modules, os.DirFS(filepath.Dir(path)))
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil || len(modules) == 0 {
|
||||||
|
panic(fmt.Sprintf("no shipped module found under ../data: %v", err))
|
||||||
|
}
|
||||||
|
return modules
|
||||||
|
}()
|
||||||
|
|
||||||
|
// svSE, enUS and misc are copies of a shipped module's files without its manifest, so a test
|
||||||
|
// layers its categories unprefixed with -d.
|
||||||
|
var svSE, enUS, misc string
|
||||||
|
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
tmp, err := os.MkdirTemp("", "fejkdata-cli-test")
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
svSE, enUS, misc = bareCopy(tmp, "sv_SE"), bareCopy(tmp, "en_US"), bareCopy(tmp, "misc")
|
||||||
|
code := m.Run()
|
||||||
|
os.RemoveAll(tmp)
|
||||||
|
os.Exit(code)
|
||||||
|
}
|
||||||
|
|
||||||
|
// bareCopy copies the category and rows files of the shipped module in data/<module> into a
|
||||||
|
// folder under tmp, and returns that folder.
|
||||||
|
func bareCopy(tmp, module string) string {
|
||||||
|
src, dst := filepath.Join("..", "data", module), filepath.Join(tmp, module)
|
||||||
|
if err := os.Mkdir(dst, 0o755); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
for _, pattern := range []string{"*.json", "*.tsv"} {
|
||||||
|
files, err := filepath.Glob(filepath.Join(src, pattern))
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
for _, f := range files {
|
||||||
|
if strings.HasPrefix(filepath.Base(f), ".") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
b, err := os.ReadFile(f)
|
||||||
|
if err == nil {
|
||||||
|
err = os.WriteFile(filepath.Join(dst, filepath.Base(f)), b, 0o644)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return dst
|
||||||
|
}
|
||||||
|
|
||||||
|
func runOut(stdin string, args ...string) (int, string, string) {
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
code := run("fejkdata", shipped, args, 1, strings.NewReader(stdin), &out, &errb)
|
||||||
|
return code, out.String(), errb.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// runTerminal runs as from a shell with nothing piped in.
|
||||||
|
func runTerminal(args ...string) (int, string, string) {
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
code := run("fejkdata", shipped, args, 1, nil, &out, &errb)
|
||||||
|
return code, out.String(), errb.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunLoadsTheModulesItIsGiven(t *testing.T) {
|
||||||
|
module := fstest.MapFS{"pet.json": {Data: []byte(`{"format":"{name}","name":["Fido"]}`)}}
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
if code := run("fejkdata", []fs.FS{module}, nil, 1, strings.NewReader("{/pet}"), &out, &errb); code != 0 || out.String() != "Fido" {
|
||||||
|
t.Errorf("run over a module holding pet = %d, %q, %q, want Fido", code, out.String(), errb.String())
|
||||||
|
}
|
||||||
|
out.Reset()
|
||||||
|
if code := run("fejkdata", []fs.FS{module}, []string{"--list"}, 1, nil, &out, &errb); code != 0 || out.String() != "pet\npet.name\n" {
|
||||||
|
t.Errorf("--list over a module holding pet = %d, %q, want pet and pet.name only", code, out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunOutputsValue(t *testing.T) {
|
||||||
|
code, out, errb := runOut("{/person}", "--data-path", svSE)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||||
|
}
|
||||||
|
if out == "" || strings.Contains(out, "\n") {
|
||||||
|
t.Fatalf("output = %q, stderr=%q; want one value and nothing added", out, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunDotPath(t *testing.T) {
|
||||||
|
code, full, _ := runOut("{/person}", "--seed", "7", "--data-path", svSE)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("person run = %d", code)
|
||||||
|
}
|
||||||
|
code, last, errb := runOut("{/person.last}", "--seed", "7", "--data-path", svSE)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("person.last run = %d, stderr=%q", code, errb)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(last) == "" {
|
||||||
|
t.Fatal("empty person.last output")
|
||||||
|
}
|
||||||
|
if last == full {
|
||||||
|
t.Errorf("person.last %q should differ from person %q", last, full)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunWithoutASeedTakesTheSystemSeed(t *testing.T) {
|
||||||
|
var drawn, seeded, errb bytes.Buffer
|
||||||
|
if code := run("fejkdata", shipped, nil, 42, strings.NewReader("{uuid()}"), &drawn, &errb); code != 0 {
|
||||||
|
t.Fatalf("run with system seed 42 = %d, stderr=%q", code, errb.String())
|
||||||
|
}
|
||||||
|
if code := run("fejkdata", shipped, []string{"--seed", "42"}, 7, strings.NewReader("{uuid()}"), &seeded, &errb); code != 0 {
|
||||||
|
t.Fatalf("run --seed 42 = %d, stderr=%q", code, errb.String())
|
||||||
|
}
|
||||||
|
if drawn.String() != seeded.String() {
|
||||||
|
t.Fatalf("run with system seed 42 = %q, run --seed 42 = %q; want the same draw", drawn.String(), seeded.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunSeedSpellings(t *testing.T) {
|
||||||
|
_, want, _ := runOut("{/address}", "--seed", "42", "--data-path", svSE)
|
||||||
|
for _, args := range [][]string{
|
||||||
|
{"--seed=42", "--data-path", svSE},
|
||||||
|
{"-s", "42", "-d", svSE},
|
||||||
|
{"--data-path", svSE, "--seed", "42"},
|
||||||
|
} {
|
||||||
|
code, got, errb := runOut("{/address}", args...)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
|
||||||
|
}
|
||||||
|
if got != want {
|
||||||
|
t.Errorf("run(%v) = %q, want %q", args, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunShortFlagValues(t *testing.T) {
|
||||||
|
_, want, _ := runOut("{/address}", "--seed", "42", "--data-path", svSE)
|
||||||
|
for _, args := range [][]string{
|
||||||
|
{"-s42", "-d", svSE},
|
||||||
|
{"-s", "42", "-d" + svSE},
|
||||||
|
{"-d", svSE, "-s42"},
|
||||||
|
} {
|
||||||
|
code, got, errb := runOut("{/address}", args...)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
|
||||||
|
}
|
||||||
|
if got != want {
|
||||||
|
t.Errorf("run(%v) = %q, want %q", args, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, three, _ := runOut("{/word}\n", "-s", "1", "-n3", "-d", svSE)
|
||||||
|
if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 {
|
||||||
|
t.Errorf("-n3 gave %d lines: %q", len(lines), three)
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
args []string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{[]string{"-s=42", "-d", svSE}, "-s42"},
|
||||||
|
{[]string{"-s=42", "-d", svSE}, "--seed=42"},
|
||||||
|
{[]string{"-d=" + svSE}, "--data-path="},
|
||||||
|
{[]string{"-nd", "3", "-d", svSE}, `--repeat needs an integer in 0..1048576, got "d"`},
|
||||||
|
{[]string{"--seed=", "-d", svSE}, `--seed needs an unsigned integer, got ""`},
|
||||||
|
} {
|
||||||
|
code, out, errb := runOut("{/word}", c.args...)
|
||||||
|
if code != 2 || out != "" {
|
||||||
|
t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(errb, c.want) {
|
||||||
|
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
code, out, _ := runOut("", "-hd", svSE)
|
||||||
|
if code != 0 || !strings.Contains(out, "Usage") {
|
||||||
|
t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunDoubleDashEndsFlags(t *testing.T) {
|
||||||
|
code, out, errb := runOut("{/person}", "--data-path", svSE, "--")
|
||||||
|
if code != 0 || strings.TrimSpace(out) == "" {
|
||||||
|
t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
|
||||||
|
}
|
||||||
|
code, _, errb = runOut("{/person}", "--data-path", svSE, "--", "--list")
|
||||||
|
if code != 2 || !strings.Contains(errb, "stdin") {
|
||||||
|
t.Errorf("after --, --list should be an argument: code %d, stderr %q", code, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunSingleDashLongIsRejected(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
args []string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{[]string{"-seed", "42", "-d", svSE}, "use --seed"},
|
||||||
|
{[]string{"-seed=42", "-d", svSE}, "use --seed"},
|
||||||
|
{[]string{"-data-path", svSE}, "use --data-path"},
|
||||||
|
{[]string{"-list", "-d", svSE}, "use --list"},
|
||||||
|
{[]string{"--nope", "-d", svSE}, "unknown flag --nope"},
|
||||||
|
{[]string{"-x", "-d", svSE}, "unknown flag -x"},
|
||||||
|
} {
|
||||||
|
code, _, errb := runOut("{/person}", c.args...)
|
||||||
|
if code != 2 {
|
||||||
|
t.Errorf("run(%v) = %d, want 2", c.args, code)
|
||||||
|
}
|
||||||
|
if !strings.Contains(errb, c.want) {
|
||||||
|
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFlagValues(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
args []string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{[]string{"--seed", "-d", svSE}, `--seed needs an unsigned integer, got "-d"`},
|
||||||
|
{[]string{"-d", svSE, "--seed"}, "--seed needs a value"},
|
||||||
|
{[]string{"--seed", "x", "-d", svSE}, "--seed"},
|
||||||
|
{[]string{"--list=1", "-d", svSE}, "--list takes no value"},
|
||||||
|
{[]string{"--repeat", "-1", "-d", svSE}, "--repeat"},
|
||||||
|
{[]string{"-n", "-1", "-d", svSE}, "--repeat"},
|
||||||
|
} {
|
||||||
|
code, _, errb := runOut("{/person}", c.args...)
|
||||||
|
if code != 2 {
|
||||||
|
t.Errorf("run(%v) = %d, want 2", c.args, code)
|
||||||
|
}
|
||||||
|
if !strings.Contains(errb, c.want) {
|
||||||
|
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRepeat(t *testing.T) {
|
||||||
|
code, out, errb := runOut("{/word}\n", "--seed", "1", "--repeat", "3", "--data-path", svSE)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||||
|
}
|
||||||
|
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
|
||||||
|
if len(lines) != 3 {
|
||||||
|
t.Errorf("repeat=3 gave %d lines: %q", len(lines), out)
|
||||||
|
}
|
||||||
|
_, short, _ := runOut("{/word}\n", "--seed", "1", "-n", "3", "-d", svSE)
|
||||||
|
if short != out {
|
||||||
|
t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunSeparator(t *testing.T) {
|
||||||
|
code, out, errb := runOut("{/word}", "--repeat", "3", "--separator", ",", "--data-path", svSE)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||||
|
}
|
||||||
|
if n := strings.Count(out, "\n"); n != 0 {
|
||||||
|
t.Errorf("want no newline, got %d: %q", n, out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, ",") {
|
||||||
|
t.Errorf("values should be comma-joined: %q", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRepeatAdvancesRNG(t *testing.T) {
|
||||||
|
code, out, errb := runOut("{/person}\n", "--seed", "1", "--repeat", "5", "--data-path", svSE)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||||
|
}
|
||||||
|
uniq := map[string]bool{}
|
||||||
|
for _, l := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
|
||||||
|
uniq[l] = true
|
||||||
|
}
|
||||||
|
if len(uniq) < 2 {
|
||||||
|
t.Errorf("repeat should vary output, all identical: %q", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunMisuse(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
stdin string
|
||||||
|
args []string
|
||||||
|
}{
|
||||||
|
{"{/person}", []string{"-d", svSE, "word"}},
|
||||||
|
{"", []string{"-d", svSE, "--list", "person"}},
|
||||||
|
{"{/sv_SE.person}", []string{"--no-shipped-data"}},
|
||||||
|
} {
|
||||||
|
code, out, errb := runOut(c.stdin, c.args...)
|
||||||
|
if code != 2 {
|
||||||
|
t.Errorf("run(%q, %v) = %d, want 2", c.stdin, c.args, code)
|
||||||
|
}
|
||||||
|
if !strings.Contains(errb, "try 'fejkdata --help'") {
|
||||||
|
t.Errorf("run(%q, %v) stderr = %q, want a pointer to --help", c.stdin, c.args, errb)
|
||||||
|
}
|
||||||
|
if out != "" {
|
||||||
|
t.Errorf("run(%q, %v) stdout = %q, want nothing", c.stdin, c.args, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunTakesNoOperand(t *testing.T) {
|
||||||
|
file := filepath.Join(t.TempDir(), "users.tmpl")
|
||||||
|
if err := os.WriteFile(file, []byte("{/sv_SE.person}"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
args []string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{[]string{"sv_SE.person"}, "echo '{/sv_SE.person}' | fejkdata"},
|
||||||
|
{[]string{"/en_US.address"}, "echo '{/en_US.address}' | fejkdata"},
|
||||||
|
{[]string{"{/en_US.person}"}, "echo '{/en_US.person}' | fejkdata"},
|
||||||
|
{[]string{"name: {/sv_SE.person.last}"}, "echo 'name: {/sv_SE.person.last}' | fejkdata"},
|
||||||
|
{[]string{"geo.US.locality[O'Fallon].name"}, `echo '{/geo.US.locality[O'\''Fallon].name}' | fejkdata`},
|
||||||
|
{[]string{"en_US.address"}, "echo '{/en_US.address}' | fejkdata"},
|
||||||
|
{[]string{`C:\temp {/sv_SE.person}`}, `printf '%s' 'C:\temp {/sv_SE.person}' | fejkdata`},
|
||||||
|
{[]string{"-"}, "drop it"},
|
||||||
|
{[]string{file}, "fejkdata < " + file},
|
||||||
|
} {
|
||||||
|
code, out, errb := runOut("", c.args...)
|
||||||
|
if code != 2 || out != "" || !strings.Contains(errb, "unexpected argument") || !strings.Contains(errb, c.want) {
|
||||||
|
t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type failingReader struct{}
|
||||||
|
|
||||||
|
func (failingReader) Read([]byte) (int, error) { return 0, errors.New("bad file descriptor") }
|
||||||
|
|
||||||
|
func TestRunUnreadableStdinIsMisuse(t *testing.T) {
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
if code := run("fejkdata", shipped, nil, 1, failingReader{}, &out, &errb); code != 2 || !strings.Contains(errb.String(), "cannot be read") {
|
||||||
|
t.Errorf("run with an unreadable stdin = %d, %q; want misuse naming it", code, errb.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunTerminalStdinIsMisuse(t *testing.T) {
|
||||||
|
code, out, errb := runTerminal("--seed", "1")
|
||||||
|
if code != 2 || out != "" || !strings.Contains(errb, "nothing is piped in: echo 'id: {int(1,999)}' | fejkdata; fejkdata --list prints the paths a {/path} reference reads\n") {
|
||||||
|
t.Errorf("run with nothing piped = %d, %q, %q; want misuse showing a template piped to fejkdata", code, out, errb)
|
||||||
|
}
|
||||||
|
for _, args := range [][]string{{"--list"}, {"--help"}, {"--version"}} {
|
||||||
|
if code, out, errb := runTerminal(args...); code != 0 || out == "" {
|
||||||
|
t.Errorf("run(%v) with nothing piped = %d, %q, %q; want it to run, reading no stdin", args, code, out, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRendersExactlyWhatIsSent(t *testing.T) {
|
||||||
|
for stdin, want := range map[string]string{
|
||||||
|
"hihi": "hihi",
|
||||||
|
"hihi\n": "hihi\n",
|
||||||
|
"hihi\r\n": "hihi\r\n",
|
||||||
|
"hihi\n\n": "hihi\n\n",
|
||||||
|
"{{x}}\n": "{x}\n",
|
||||||
|
"\"a\"\n": "a",
|
||||||
|
" \t\"a\"\r\n\n": "a",
|
||||||
|
"{/misc.territory[SE].alpha2}": "SE",
|
||||||
|
"{/misc.territory[SE].alpha2}\n": "SE\n",
|
||||||
|
} {
|
||||||
|
code, out, errb := runOut(stdin, "--seed", "1")
|
||||||
|
if code != 0 || out != want {
|
||||||
|
t.Errorf("run(%q) = %d, %q, %q; want %q", stdin, code, out, errb, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if code, out, errb := runOut("{\"format\":\"{x}\\n\",\"x\":[\"a\",\"b\"]}\n"); code != 0 || (out != "a\n" && out != "b\n") {
|
||||||
|
t.Errorf("a JSON template's format = %d, %q, %q; want its own newline only", code, out, errb)
|
||||||
|
}
|
||||||
|
if code, out, errb := runOut("{/sv_SE.word}\n", "--seed", "1", "-n", "3", "--separator", ","); code != 0 || strings.Count(out, "\n,") != 2 || !strings.HasSuffix(out, "\n") {
|
||||||
|
t.Errorf("echo's newline and a separator = %d, %q, %q; want each render's newline, then the separator", code, out, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRendersBlankStdin(t *testing.T) {
|
||||||
|
for _, stdin := range []string{"", "\n", " \r\n\t"} {
|
||||||
|
if code, out, errb := runOut(stdin); code != 0 || out != stdin {
|
||||||
|
t.Errorf("run(%q) = %d, %q, %q; want the blank template rendered as it is", stdin, code, out, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunTextPrintsAsWritten(t *testing.T) {
|
||||||
|
for _, stdin := range []string{"sv_SE.person", "[Skåne län]", "x[1]y", `say "hi"`} {
|
||||||
|
code, out, errb := runOut(stdin)
|
||||||
|
if code != 0 || out != stdin {
|
||||||
|
t.Errorf("run(%q) = %d, %q, %q; want the text as written", stdin, code, out, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunMultipleDataPaths(t *testing.T) {
|
||||||
|
code, out, errb := runOut("{/person}", "-d", enUS, "--data-path", svSE)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(out) == "" {
|
||||||
|
t.Fatal("empty output for multi-dir run")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunUnknownCategoryFails(t *testing.T) {
|
||||||
|
code, _, errb := runOut("{/nope}", "--data-path", svSE, "--format", "csv")
|
||||||
|
if code != 1 {
|
||||||
|
t.Fatalf("run = %d, want 1", code)
|
||||||
|
}
|
||||||
|
if !strings.Contains(errb, "nope") {
|
||||||
|
t.Errorf("stderr %q should name the unknown category", errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunList(t *testing.T) {
|
||||||
|
code, out, errb := runOut("", "--data-path", svSE, "--list")
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"person", "person.last", "address"} {
|
||||||
|
if !strings.Contains(out, want) {
|
||||||
|
t.Errorf("list output missing %q:\n%s", want, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunHelpOnStdout(t *testing.T) {
|
||||||
|
for _, h := range []string{"-h", "--help"} {
|
||||||
|
code, out, errb := runOut("", h)
|
||||||
|
if code != 0 {
|
||||||
|
t.Errorf("%s = %d, want 0", h, code)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "Usage") || errb != "" {
|
||||||
|
t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVersionLineNamesFejkdatasVersion(t *testing.T) {
|
||||||
|
fejkdata := func(version string, replace *debug.Module) *debug.Module {
|
||||||
|
return &debug.Module{Path: "github.com/larvit/fejkdata", Version: version, Replace: replace}
|
||||||
|
}
|
||||||
|
mycli := debug.Module{Path: "example.com/mycli", Version: "v9.0.0"}
|
||||||
|
for _, c := range []struct {
|
||||||
|
info *debug.BuildInfo
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{&debug.BuildInfo{Main: *fejkdata("v0.1.0", nil)}, "fejkdata v0.1.0"},
|
||||||
|
{&debug.BuildInfo{Main: *fejkdata("(devel)", nil)}, "fejkdata devel"},
|
||||||
|
{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", nil)}}, "mycli v9.0.0, fejkdata v0.1.0"},
|
||||||
|
{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", &debug.Module{Path: "../fejkdata"})}}, "mycli v9.0.0, fejkdata devel"},
|
||||||
|
{&debug.BuildInfo{Main: mycli, Deps: []*debug.Module{fejkdata("v0.1.0", &debug.Module{Path: "example.com/fork", Version: "v0.1.1"})}}, "mycli v9.0.0, fejkdata v0.1.1"},
|
||||||
|
{&debug.BuildInfo{Main: mycli}, "mycli v9.0.0, fejkdata devel"},
|
||||||
|
{nil, "fejkdata devel"},
|
||||||
|
} {
|
||||||
|
if got := versionLine("mycli", c.info); got != c.want {
|
||||||
|
t.Errorf("versionLine(mycli, %+v) = %q, want %q", c.info, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommandNameIsTheFileItRunsFrom(t *testing.T) {
|
||||||
|
for arg0, want := range map[string]string{
|
||||||
|
"/usr/local/bin/mycli": "mycli",
|
||||||
|
"mycli.EXE": "mycli",
|
||||||
|
"mycli.exe": "mycli",
|
||||||
|
"my.cli": "my.cli",
|
||||||
|
"": "fejkdata",
|
||||||
|
} {
|
||||||
|
if got := commandName(arg0); got != want {
|
||||||
|
t.Errorf("commandName(%q) = %q, want %q", arg0, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunNamesItselfByTheCommandName(t *testing.T) {
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
if code := run("mycli", shipped, []string{"--nope"}, 1, nil, &out, &errb); code != 2 || errb.String() != "mycli: unknown flag --nope\ntry 'mycli --help'\n" {
|
||||||
|
t.Errorf("mycli --nope = %d, %q", code, errb.String())
|
||||||
|
}
|
||||||
|
errb.Reset()
|
||||||
|
if code := run("mycli", shipped, nil, 1, nil, &out, &errb); code != 2 || !strings.Contains(errb.String(), "| mycli; mycli --list prints") {
|
||||||
|
t.Errorf("mycli with nothing piped = %d, %q, want a pipe into mycli", code, errb.String())
|
||||||
|
}
|
||||||
|
errb.Reset()
|
||||||
|
if code := run("mycli", []fs.FS{nil, nil}, nil, 1, strings.NewReader("x"), &out, &errb); code != 1 || strings.Count(errb.String(), "mycli: ") != 2 || strings.Contains(errb.String(), "fejkdata") {
|
||||||
|
t.Errorf("mycli over two nil modules = %d, %q, want each line under mycli", code, errb.String())
|
||||||
|
}
|
||||||
|
errb.Reset()
|
||||||
|
if code := run("mycli", shipped, []string{"-d", "/nope"}, 1, strings.NewReader("{/sv_SE.person}"), &out, &errb); code != 1 || !strings.HasPrefix(errb.String(), "mycli: ") || strings.Contains(errb.String(), "fejkdata") {
|
||||||
|
t.Errorf("mycli -d /nope = %d, %q, want a load error under mycli", code, errb.String())
|
||||||
|
}
|
||||||
|
if code := run("mycli", shipped, []string{"--help"}, 1, nil, &out, &errb); code != 0 || !strings.HasPrefix(out.String(), "Usage: <template> | mycli [flags]\n") || strings.Contains(out.String(), "fejkdata") {
|
||||||
|
t.Errorf("mycli --help = %d, %q, want mycli named throughout", code, out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunVersion(t *testing.T) {
|
||||||
|
code, out, errb := runOut("", "--version")
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("--version = %d, stderr=%q", code, errb)
|
||||||
|
}
|
||||||
|
version, ok := strings.CutPrefix(out, "fejkdata ")
|
||||||
|
if !ok || strings.TrimSpace(version) == "" {
|
||||||
|
t.Errorf("--version = %q, want the command name and a version on stdout", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunMissingDirFails(t *testing.T) {
|
||||||
|
code, _, errb := runOut("{/person}", "--data-path", "../data/nope")
|
||||||
|
if code != 1 {
|
||||||
|
t.Fatalf("run = %d, want 1", code)
|
||||||
|
}
|
||||||
|
if errb == "" {
|
||||||
|
t.Error("want an error message on stderr")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunShippedDataByDefault(t *testing.T) {
|
||||||
|
code, out, errb := runOut("{/sv_SE.person}", "--seed", "1")
|
||||||
|
if code != 0 || strings.TrimSpace(out) == "" {
|
||||||
|
t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
|
||||||
|
}
|
||||||
|
code, list, _ := runOut("", "--list")
|
||||||
|
if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") {
|
||||||
|
t.Errorf("--list without --data-path = %d, %q", code, list)
|
||||||
|
}
|
||||||
|
code, _, errb = runOut("{/person}")
|
||||||
|
if code != 2 || !strings.Contains(errb, "person") {
|
||||||
|
t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunSelectsATableRow(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
for name, content := range map[string]string{
|
||||||
|
"region.json": `{"format":"{name}","rows":"region.tsv","key":"code","name":"name"}`,
|
||||||
|
"region.tsv": "code\tname\n01\tStockholms län\n12\tSkåne län\n",
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if code, out, errb := runOut("{/region[12]}", "--no-shipped-data", "-d", dir); code != 0 || out != "Skåne län" {
|
||||||
|
t.Fatalf("{/region[12]} = %d, %q, stderr=%q", code, out, errb)
|
||||||
|
}
|
||||||
|
if code, out, _ := runOut("{/region[Skåne län].code}", "--no-shipped-data", "-d", dir); code != 0 || out != "12" {
|
||||||
|
t.Fatalf("{/region[Skåne län].code} = %d, %q", code, out)
|
||||||
|
}
|
||||||
|
if code, out, _ := runOut("{/region[12]}", "--no-shipped-data", "-d", dir, "--format", "sql"); code != 0 || out != `INSERT INTO "region" ("code", "name") VALUES ('12', 'Skåne län');`+"\n" {
|
||||||
|
t.Fatalf("--format sql {/region[12]} = %d, %q, want the table named without its selector", code, out)
|
||||||
|
}
|
||||||
|
if code, _, errb := runOut("{/region[99]}", "--no-shipped-data", "-d", dir); code != 2 || !strings.Contains(errb, `"99"`) {
|
||||||
|
t.Fatalf("{/region[99]} = %d, stderr=%q, want misuse naming the row", code, errb)
|
||||||
|
}
|
||||||
|
if code, out, _ := runOut("{/misc.territory[SE]}", "--seed", "1", "--format", "csv"); code != 0 || !strings.HasPrefix(out, "alpha2,") || !strings.Contains(out, "\nSE,SWE,") {
|
||||||
|
t.Fatalf("--format csv {/misc.territory[SE]} = %d, %q", code, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUsageExamplesRenderWithNoData(t *testing.T) {
|
||||||
|
examples := regexp.MustCompile(`(?m)^ echo '([^']*)' \| COMMAND$`).FindAllStringSubmatch(usage+"\n "+pipeHint("COMMAND")+"\n", -1)
|
||||||
|
if len(examples) < 3 {
|
||||||
|
t.Fatalf("usage and pipe hint hold %d echo examples, want the help's two and the hint's", len(examples))
|
||||||
|
}
|
||||||
|
for _, e := range examples {
|
||||||
|
if code, out, errb := runOut(e[1], "--no-shipped-data"); code != 0 || strings.TrimSpace(out) == "" {
|
||||||
|
t.Errorf("echo %q with no data: run = %d, %q, stderr %q", e[1], code, out, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, token := range regexp.MustCompile(`\{/[^}]*\}`).FindAllString(usage, -1) {
|
||||||
|
if token != "{/path}" && token != "{/table[KEY]}" {
|
||||||
|
t.Errorf("usage names %s, a path a custom CLI may not carry", token)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(usage, "\nExit codes: 0 success, 1 runtime error, 2 misuse.\n") {
|
||||||
|
t.Errorf("usage does not list the exit codes")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunOperandMisuseBeforeLoad(t *testing.T) {
|
||||||
|
code, _, errb := runOut("", "--no-shipped-data", "x")
|
||||||
|
if code != 2 || !strings.Contains(errb, "unexpected argument") || strings.Contains(errb, "--data-path") {
|
||||||
|
t.Fatalf("an operand with no data = %d, %q; want the operand refused before any load", code, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunInlineTemplate(t *testing.T) {
|
||||||
|
code, out, errb := runOut("name: {/sv_SE.person.last}", "--seed", "1")
|
||||||
|
if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
|
||||||
|
t.Fatalf("inline format string = %d, %q, stderr %q", code, out, errb)
|
||||||
|
}
|
||||||
|
code, out, errb = runOut(`{"format":"name: {x}","x":["bosse","lina"]}`, "--seed", "1")
|
||||||
|
if code != 0 || (out != "name: bosse" && out != "name: lina") {
|
||||||
|
t.Fatalf("inline JSON template = %d, %q, want one name, stderr %q", code, out, errb)
|
||||||
|
}
|
||||||
|
code, out, errb = runOut(`"name: {/sv_SE.person.last}"`, "--seed", "1")
|
||||||
|
if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
|
||||||
|
t.Fatalf("inline JSON string = %d, %q, stderr %q", code, out, errb)
|
||||||
|
}
|
||||||
|
code, out, errb = runOut("{digits(1)}\n", "--seed", "1", "-n", "2")
|
||||||
|
if code != 0 || len(strings.Split(strings.TrimRight(out, "\n"), "\n")) != 2 {
|
||||||
|
t.Fatalf("inline template with --repeat = %d, %q, stderr %q", code, out, errb)
|
||||||
|
}
|
||||||
|
code, out, errb = runOut("{/sv_SE.person} hihi", "--seed", "1")
|
||||||
|
if code != 0 || !strings.HasSuffix(out, " hihi") || strings.Contains(out, "{") {
|
||||||
|
t.Fatalf("a reference and text = %d, %q, stderr %q", code, out, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFormatTakesOnlyALoneReferenceAsARecord(t *testing.T) {
|
||||||
|
for _, stdin := range []string{"{//sv_SE.person}", "{/sv_SE.person|/misc.uuid}", "{/sv_SE.person as p}", "{/}", "{/.sv_SE.person}"} {
|
||||||
|
if code, out, errb := runOut(stdin, "--format", "json"); code != 2 || out != "" {
|
||||||
|
t.Errorf("run(%q, --format json) = %d, %q, %q; want misuse", stdin, code, out, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if code, out, errb := runOut(`{"format":"x"}`); code != 0 || out != "x" {
|
||||||
|
t.Errorf(`run({"format":"x"}) = %d, %q, %q; want x`, code, out, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunTemplateMisuse(t *testing.T) {
|
||||||
|
for stdin, want := range map[string]string{
|
||||||
|
"{bad": "unterminated",
|
||||||
|
"name: {/no.such.path}": "no entry",
|
||||||
|
"a } b": "}}",
|
||||||
|
"{//sv_SE.person}": "write {/sv_SE.person}",
|
||||||
|
} {
|
||||||
|
code, out, errb := runOut(stdin, "--seed", "1")
|
||||||
|
if code != 2 || out != "" || !strings.Contains(errb, "try 'fejkdata --help'") || !strings.Contains(errb, want) {
|
||||||
|
t.Errorf("run(%q) = %d, %q, %q; want misuse naming %q and --help", stdin, code, out, errb, want)
|
||||||
|
}
|
||||||
|
if strings.Contains(errb, "fejkdata: fejkdata:") {
|
||||||
|
t.Errorf("run(%q) doubled the program prefix: %q", stdin, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunNoShippedData(t *testing.T) {
|
||||||
|
code, list, errb := runOut("", "--no-shipped-data", "-d", misc, "--list")
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||||
|
}
|
||||||
|
if strings.Contains(list, "sv_SE") || !strings.Contains(list, "uuid\n") {
|
||||||
|
t.Errorf("--no-shipped-data --list = %q, want only the given dir", list)
|
||||||
|
}
|
||||||
|
code, out, _ := runOut("{/uuid}", "--no-shipped-data", "-d", misc, "-s", "3")
|
||||||
|
if code != 0 || strings.TrimSpace(out) == "" {
|
||||||
|
t.Errorf("run = %d, out=%q", code, out)
|
||||||
|
}
|
||||||
|
if code, out, errb := runOut("{digits(3)}", "--no-shipped-data"); code != 0 || len(out) != 3 {
|
||||||
|
t.Errorf("--no-shipped-data alone = %d, %q, %q, want a template reading no data rendered", code, out, errb)
|
||||||
|
}
|
||||||
|
if code, _, errb := runOut("{/sv_SE.person}", "--no-shipped-data"); code != 2 || !strings.HasSuffix(errb, "no data is loaded\ntry 'fejkdata --help'\n") {
|
||||||
|
t.Errorf("--no-shipped-data alone = %d, %q, want a reference to nothing refused", code, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRepeatOfZeroWritesNoRecord(t *testing.T) {
|
||||||
|
dir := recordDir(t)
|
||||||
|
for _, c := range []struct {
|
||||||
|
args []string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{nil, ""},
|
||||||
|
{[]string{"--format", "json"}, "[]\n"},
|
||||||
|
{[]string{"--format", "ndjson"}, ""},
|
||||||
|
{[]string{"--format", "csv"}, "first,last\n"},
|
||||||
|
{[]string{"--format", "sql"}, ""},
|
||||||
|
} {
|
||||||
|
if code, out, errb := runOut("{/users}", append(c.args, "-n", "0", "-d", dir)...); code != 0 || out != c.want {
|
||||||
|
t.Errorf("run(-n 0 %v) = %d, %q, %q, want %q", c.args, code, out, errb, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if code, _, _ := runOut("{/nope}", "-n", "0", "--format", "json", "-d", dir); code != 1 {
|
||||||
|
t.Errorf("run(-n 0) of a reference to nothing = %d, want 1", code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRepeatIsBounded(t *testing.T) {
|
||||||
|
for _, args := range [][]string{
|
||||||
|
{"--repeat", "9223372036854775807"},
|
||||||
|
{"--repeat", "1048577"},
|
||||||
|
{"-n", "99999999999"},
|
||||||
|
} {
|
||||||
|
code, out, errb := runOut("{/sv_SE.word}", args...)
|
||||||
|
if code != 2 || out != "" || !strings.Contains(errb, "0..1048576") {
|
||||||
|
t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
code, out, errb := runOut("{/x}", "--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "")
|
||||||
|
if code != 0 || len(out) != 1048576 {
|
||||||
|
t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunIgnoresAFlagWithNoEffect(t *testing.T) {
|
||||||
|
dir := recordDir(t)
|
||||||
|
for _, c := range []struct {
|
||||||
|
stdin string
|
||||||
|
base, ignored []string
|
||||||
|
}{
|
||||||
|
{"", []string{"--list"}, []string{"-n", "3"}},
|
||||||
|
{"", []string{"--list"}, []string{"--separator", ","}},
|
||||||
|
{"", []string{"--list"}, []string{"--format", "json"}},
|
||||||
|
{"", []string{"--list"}, []string{"--table", "t"}},
|
||||||
|
{"{/users}", nil, []string{"--table", "t"}},
|
||||||
|
{"{/users}", []string{"--format", "json"}, []string{"--table", "t"}},
|
||||||
|
} {
|
||||||
|
args := append([]string{"--seed", "1", "--data-path", dir}, c.base...)
|
||||||
|
_, want, _ := runOut(c.stdin, args...)
|
||||||
|
code, got, errb := runOut(c.stdin, append(args, c.ignored...)...)
|
||||||
|
if code != 0 || want == "" || got != want {
|
||||||
|
t.Errorf("run(%v + %v) = %d, %q, %q; want what %v alone prints, %q", c.base, c.ignored, code, got, errb, c.base, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunReadsABareNumberAsAFormatString(t *testing.T) {
|
||||||
|
if code, out, errb := runOut("42\n", "-n", "3"); code != 0 || out != "42\n42\n42\n" {
|
||||||
|
t.Errorf("echo 42 | fejkdata -n 3 = %d, %q, %q, want 42 on three lines", code, out, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRendersAnyJSONValue(t *testing.T) {
|
||||||
|
for stdin, want := range map[string]string{"42": "42", "42\n": "42\n", "true\n": "true\n", " 7 ": " 7 ", "null\n": "", " null ": ""} {
|
||||||
|
if code, out, errb := runOut(stdin); code != 0 || out != want {
|
||||||
|
t.Errorf("run(%q) = %d, %q, %q, want %q", stdin, code, out, errb, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunUnknownFlagIsNamedByRune(t *testing.T) {
|
||||||
|
code, _, errb := runOut("{/sv_SE.word}", "-ä")
|
||||||
|
if code != 2 || !strings.Contains(errb, "unknown flag -ä") {
|
||||||
|
t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunEmptyDataPathIsNamed(t *testing.T) {
|
||||||
|
code, _, errb := runOut("{/sv_SE.word}", "--data-path=")
|
||||||
|
if code != 1 || !strings.Contains(errb, "empty") {
|
||||||
|
t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func recordDir(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
content := `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"]}`
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "users.json"), []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return dir
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRecordJSON(t *testing.T) {
|
||||||
|
code, out, errb := runOut("{/users}", "--seed", "1", "--format", "json", "--repeat", "2", "--data-path", recordDir(t))
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||||
|
}
|
||||||
|
var arr []map[string]string
|
||||||
|
if err := json.Unmarshal([]byte(out), &arr); err != nil {
|
||||||
|
t.Fatalf("json output is not one JSON array: %v\n%q", err, out)
|
||||||
|
}
|
||||||
|
if len(arr) != 2 || len(arr[0]) != 2 || arr[0]["first"] == "" || arr[0]["last"] == "" {
|
||||||
|
t.Fatalf("json output = %q, want an array of two records with first and last columns", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRecordNDJSON(t *testing.T) {
|
||||||
|
code, out, errb := runOut("{/users}", "--seed", "1", "--format", "ndjson", "--repeat", "2", "--data-path", recordDir(t))
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||||
|
}
|
||||||
|
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
|
||||||
|
if len(lines) != 2 {
|
||||||
|
t.Fatalf("ndjson output = %d lines %q, want one object per line", len(lines), out)
|
||||||
|
}
|
||||||
|
for _, line := range lines {
|
||||||
|
var m map[string]string
|
||||||
|
if err := json.Unmarshal([]byte(line), &m); err != nil || len(m) != 2 {
|
||||||
|
t.Fatalf("ndjson line %q is not one object: %v", line, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRecordCSVRoundTrips(t *testing.T) {
|
||||||
|
code, out, errb := runOut("{/users}\r\n", "--seed", "1", "--format", "csv", "--repeat", "3", "--data-path", recordDir(t))
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||||
|
}
|
||||||
|
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
|
||||||
|
if len(lines) != 4 {
|
||||||
|
t.Fatalf("csv output = %d lines %q, want a header and 3 rows", len(lines), out)
|
||||||
|
}
|
||||||
|
if lines[0] != "first,last" {
|
||||||
|
t.Fatalf("csv header = %q, want first,last", lines[0])
|
||||||
|
}
|
||||||
|
r, err := csv.NewReader(strings.NewReader(out)).ReadAll()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("csv output is not valid CSV: %v", err)
|
||||||
|
}
|
||||||
|
if len(r) != 4 || len(r[0]) != 2 {
|
||||||
|
t.Fatalf("csv parsed to %v, want 4 records of 2 fields", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRecordSQL(t *testing.T) {
|
||||||
|
code, out, errb := runOut("{/users}", "--seed", "1", "--format", "sql", "--repeat", "2", "--table", "people", "--data-path", recordDir(t))
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||||
|
}
|
||||||
|
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
|
||||||
|
if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) {
|
||||||
|
t.Fatalf("sql output = %q, want two INSERT INTO people statements", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRecordDefaultTable(t *testing.T) {
|
||||||
|
code, out, errb := runOut("{/users}", "--seed", "1", "--format", "sql", "--data-path", recordDir(t))
|
||||||
|
if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
|
||||||
|
t.Fatalf("default table = %d, %q, want the reference's last segment; stderr %q", code, out, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRecordOfAQuotedLoneReference(t *testing.T) {
|
||||||
|
code, out, errb := runOut(`"{/users}"`, "--seed", "1", "--format", "sql", "--data-path", recordDir(t))
|
||||||
|
if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
|
||||||
|
t.Fatalf("a JSON string holding {/users} = %d, %q, %q; want the record users", code, out, errb)
|
||||||
|
}
|
||||||
|
code, out, errb = runOut("{/users} ", "--format", "csv", "--data-path", recordDir(t))
|
||||||
|
if code != 2 || out != "" || !strings.Contains(errb, "whitespace around {/users}") {
|
||||||
|
t.Errorf("{/users} and a space = %d, %q, %q; want misuse naming the whitespace", code, out, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRecordInlineTemplate(t *testing.T) {
|
||||||
|
code, out, errb := runOut(`{"format":"{x}","x":["a","b"]}`, "--seed", "1", "--format", "json")
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("inline record = %d, stderr=%q", code, errb)
|
||||||
|
}
|
||||||
|
var arr []map[string]string
|
||||||
|
if err := json.Unmarshal([]byte(out), &arr); err != nil || len(arr) != 1 || (arr[0]["x"] != "a" && arr[0]["x"] != "b") {
|
||||||
|
t.Fatalf("inline record json = %q, want one record with a column x (err %v)", out, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRecordMisuse(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
args []string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{[]string{"--format", "yaml"}, "--format takes csv, json, ndjson, sql or text"},
|
||||||
|
{[]string{"--format", "sql", "--table", ""}, "--table names the INSERT target"},
|
||||||
|
{[]string{"--format", "csv", "--separator", ";"}, `--separator joins text renders; --format csv always separates fields with ","`},
|
||||||
|
{[]string{"--format", "json", "--separator", ","}, "--separator joins text renders"},
|
||||||
|
{[]string{"--format", "ndjson", "--separator", ","}, "--separator joins text renders"},
|
||||||
|
{[]string{"--format", "sql", "--separator", ","}, "--separator joins text renders"},
|
||||||
|
} {
|
||||||
|
code, out, errb := runOut("{/users}", c.args...)
|
||||||
|
if code != 2 || out != "" || !strings.Contains(errb, c.want) {
|
||||||
|
t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRecordOnABareValueIsRuntimeError(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "word.json"), []byte(`["a","b"]`), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
code, _, errb := runOut("{/word}", "--format", "json", "--data-path", dir)
|
||||||
|
if code != 1 {
|
||||||
|
t.Fatalf("record on a choice = %d, want exit 1 (runtime error), stderr %q", code, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRecordRejectsATopLevelRepeat(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
content := `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "rep.json"), []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
code, _, errb := runOut("{/rep}", "--format", "json", "--data-path", dir)
|
||||||
|
if code != 1 || !strings.Contains(errb, "carries repeat 3") {
|
||||||
|
t.Errorf("record on a repeating path = %d, %q; want exit 1 naming the repeat", code, errb)
|
||||||
|
}
|
||||||
|
code, _, errb = runOut(content, "--format", "json")
|
||||||
|
if code != 2 || !strings.Contains(errb, "carries repeat 3") {
|
||||||
|
t.Errorf("record on a repeating inline template = %d, %q; want exit 2 naming the repeat", code, errb)
|
||||||
|
}
|
||||||
|
if code, out, _ := runOut("{/rep}", "--seed", "1", "--data-path", dir); code != 0 || strings.TrimSpace(out) != "x-|x-|x-" {
|
||||||
|
t.Errorf("text view = %d, %q; want the repeat still composed", code, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunNumbersFollowTheShell(t *testing.T) {
|
||||||
|
_, want, _ := runOut("{/sv_SE.word}", "--seed", "7", "--repeat", "3")
|
||||||
|
code, got, errb := runOut("{/sv_SE.word}", "--seed", "007", "--repeat", "+3")
|
||||||
|
if code != 0 || got != want {
|
||||||
|
t.Errorf("run(--seed 007 --repeat +3) = %d, %q, stderr %q; want the same as --seed 7 --repeat 3 %q", code, got, errb, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunRecordOfAFieldlessPath pins the way out of asking a category of one value
|
||||||
|
// for columns: the record to write, and the quoting a shell needs to take it. The
|
||||||
|
// template is well-formed and only the data cannot serve it, so the code is 1.
|
||||||
|
func TestRunRecordOfAFieldlessPath(t *testing.T) {
|
||||||
|
code, out, errb := runOut("{/sv_SE.personnummer}", "--format", "csv")
|
||||||
|
if code != 1 {
|
||||||
|
t.Fatalf("run(--format csv) of {/sv_SE.personnummer} = %d, want 1", code)
|
||||||
|
}
|
||||||
|
for _, want := range []string{`{"format":"","personnummer":"{/sv_SE.personnummer}"}`, "single quotes"} {
|
||||||
|
if !strings.Contains(errb, want) {
|
||||||
|
t.Errorf("stderr = %q, want it to name %s", errb, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if out != "" {
|
||||||
|
t.Errorf("stdout = %q, want nothing", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestADataPathCategoryFailingAtFirstReachExits1(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
for name, data := range map[string]string{
|
||||||
|
".fejkdata.json": `{"index": {"bad": {"paths": [""]}}}`,
|
||||||
|
"bad.json": `"{/nope}"`,
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, args := range [][]string{{"-d", dir}, {"-d", dir, "--format", "json"}} {
|
||||||
|
if code, _, errb := runOut("{/bad}", args...); code != 1 || !strings.Contains(errb, "{/nope}") {
|
||||||
|
t.Errorf("%v on a category failing to load = %d, %q; want exit 1 naming the reference", args, code, errb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata/internal/grammar"
|
||||||
|
)
|
||||||
|
|
||||||
|
// grammarLoneReference is the path a format reads when the grammar parses it as one path
|
||||||
|
// read of one reference from the root, and nothing else.
|
||||||
|
func grammarLoneReference(text string) (string, bool) {
|
||||||
|
toks, err := grammar.ParseFormat(text)
|
||||||
|
if err != nil || len(toks) != 1 || toks[0].Kind != grammar.PathRead || len(toks[0].Arms) != 1 {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
sigil, path, err := grammar.RefShape(toks[0].Arms[0])
|
||||||
|
if err != nil || sigil != "/" {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return path, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoneReferenceAgreesWithTheGrammar(t *testing.T) {
|
||||||
|
for _, text := range []string{
|
||||||
|
" {/x}",
|
||||||
|
"x",
|
||||||
|
"{.x}",
|
||||||
|
"{/.x}",
|
||||||
|
"{//x}",
|
||||||
|
"{/a...b}",
|
||||||
|
"{/a..b}",
|
||||||
|
"{/a..}",
|
||||||
|
"{/a.[x]}",
|
||||||
|
"{/a.}",
|
||||||
|
"{/a|/b}",
|
||||||
|
"{/a}{/b}",
|
||||||
|
"{/misc.currency[US Dollar (Next day)]}",
|
||||||
|
"{/sv_SE.person}",
|
||||||
|
"{/t[a as b].c}",
|
||||||
|
"{/t[f(x)]}",
|
||||||
|
"{/x as p}",
|
||||||
|
"{/x(1)}",
|
||||||
|
"{/x}}",
|
||||||
|
"{/}",
|
||||||
|
} {
|
||||||
|
path, lone := loneReference(text)
|
||||||
|
want, wantLone := grammarLoneReference(text)
|
||||||
|
if lone != wantLone || path != want {
|
||||||
|
t.Errorf("loneReference(%q) = %q, %v; the grammar reads %q, %v", text, path, lone, want, wantLone)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+5
-505
@@ -1,512 +1,12 @@
|
|||||||
// Command fejkdata prints fake values from the shipped data and any directories
|
// Command fejkdata renders the template on its stdin with fake values from every shipped
|
||||||
// layered over it.
|
// module; package cli documents it.
|
||||||
//
|
|
||||||
// fejkdata sv_SE.person # a full person
|
|
||||||
// fejkdata sv_SE.person.last # just the surname
|
|
||||||
// fejkdata --data-path ./mydata sv_SE.person # layer custom data; the last dir wins
|
|
||||||
// fejkdata --seed 42 sv_SE.address
|
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
"os"
|
||||||
"runtime/debug"
|
|
||||||
"sort"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.larvit.se/larvit/fejkdata"
|
"github.com/larvit/fejkdata/cli"
|
||||||
|
"github.com/larvit/fejkdata/data"
|
||||||
)
|
)
|
||||||
|
|
||||||
const usage = `Usage: fejkdata [flags] <path|template>
|
func main() { os.Exit(cli.Main(data.Modules()...)) }
|
||||||
|
|
||||||
<path> a category, or a dotted path into one (person, person.last);
|
|
||||||
a table's row by key or name: 'misc.territory[SE]', 'misc.territory[Sweden].tld'
|
|
||||||
<template> a format string or JSON value to render inline, e.g.
|
|
||||||
'name: {/sv_SE.person.last}' or '{"format":"{x}","x":["bosse","lina"]}'
|
|
||||||
|
|
||||||
A layout inside a template is single-quoted, so quote the whole argument with " to
|
|
||||||
keep it: "{date(1990-01-01,2010-12-31,'2006-01-02')}".
|
|
||||||
|
|
||||||
An argument containing a { token, or a JSON object, array or string, is a
|
|
||||||
template; any other argument is a path (a path never contains a brace or a quote,
|
|
||||||
and a bracket only as a [selector] after a table's name). Templates reach the
|
|
||||||
data by reference from the root — {/sv_SE.person.last} — whether the data is
|
|
||||||
shipped or layered with --data-path. An argument carrying a closing brace or a
|
|
||||||
quote but no valid JSON names neither.
|
|
||||||
|
|
||||||
With --format json, ndjson, csv or sql the argument must name a record — a
|
|
||||||
template whose fields are its columns — and the rows are written as one JSON
|
|
||||||
array, one JSON object per line, one CSV row (after a header), or one INSERT.
|
|
||||||
|
|
||||||
-d, --data-path D a data directory to layer over the shipped data (repeatable; last wins on a clash)
|
|
||||||
--format F output form: text (default), json, ndjson, csv or sql
|
|
||||||
-h, --help print this help, then exit
|
|
||||||
--list list the paths the data offers, then exit
|
|
||||||
--no-shipped-data load only the --data-path directories
|
|
||||||
-n, --repeat N render the value N times, 1..1048576 (default 1)
|
|
||||||
-s, --seed N seed for reproducible output
|
|
||||||
--separator S string between repeated values (default newline)
|
|
||||||
--table T the INSERT target for --format sql (default: the path's last segment, or records for an inline template)
|
|
||||||
--version print the version, then exit
|
|
||||||
|
|
||||||
Flags may come before or after <path|template>; -- ends the flags. A short flag's
|
|
||||||
value attaches or follows (-n3, -n 3); short flags bundle (-hn 3).
|
|
||||||
`
|
|
||||||
|
|
||||||
type invocation struct {
|
|
||||||
dirs []string
|
|
||||||
format string
|
|
||||||
formatSet bool
|
|
||||||
help bool
|
|
||||||
list bool
|
|
||||||
noShipped bool
|
|
||||||
paths []string
|
|
||||||
repeat int
|
|
||||||
repeatSet bool
|
|
||||||
seed uint64
|
|
||||||
seeded bool
|
|
||||||
separator string
|
|
||||||
separatorSet bool
|
|
||||||
table string
|
|
||||||
tableSet bool
|
|
||||||
version bool
|
|
||||||
}
|
|
||||||
|
|
||||||
type flagDef struct {
|
|
||||||
long string
|
|
||||||
short string
|
|
||||||
value bool
|
|
||||||
set func(*invocation, string) error
|
|
||||||
}
|
|
||||||
|
|
||||||
var flagDefs = []flagDef{
|
|
||||||
{"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }},
|
|
||||||
{"format", "", true, func(in *invocation, v string) error { in.format, in.formatSet = v, true; return nil }},
|
|
||||||
{"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }},
|
|
||||||
{"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }},
|
|
||||||
{"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }},
|
|
||||||
{"repeat", "n", true, func(in *invocation, v string) error {
|
|
||||||
n, err := strconv.Atoi(v)
|
|
||||||
if err != nil || n < 1 || n > fejkdata.MaxRepeat {
|
|
||||||
return fmt.Errorf("--repeat needs an integer in 1..%d, got %q", fejkdata.MaxRepeat, v)
|
|
||||||
}
|
|
||||||
in.repeat, in.repeatSet = n, true
|
|
||||||
return nil
|
|
||||||
}},
|
|
||||||
{"seed", "s", true, func(in *invocation, v string) error {
|
|
||||||
n, err := strconv.ParseUint(v, 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("--seed needs an unsigned integer, got %q", v)
|
|
||||||
}
|
|
||||||
in.seed, in.seeded = n, true
|
|
||||||
return nil
|
|
||||||
}},
|
|
||||||
{"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }},
|
|
||||||
{"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }},
|
|
||||||
{"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }},
|
|
||||||
}
|
|
||||||
|
|
||||||
func flagByLong(name string) *flagDef {
|
|
||||||
for i := range flagDefs {
|
|
||||||
if flagDefs[i].long == name {
|
|
||||||
return &flagDefs[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func flagByShort(name string) *flagDef {
|
|
||||||
for i := range flagDefs {
|
|
||||||
if flagDefs[i].short == name {
|
|
||||||
return &flagDefs[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// flagArg is one flag as written: its definition and the value attached to it,
|
|
||||||
// if any.
|
|
||||||
type flagArg struct {
|
|
||||||
def *flagDef
|
|
||||||
value string
|
|
||||||
inline bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// splitFlags reads one argv element as flags: --name or --name=value, or -abc
|
|
||||||
// where each letter is a short flag and the first that takes a value takes the
|
|
||||||
// rest of the element.
|
|
||||||
func splitFlags(arg string) ([]flagArg, error) {
|
|
||||||
if strings.HasPrefix(arg, "--") {
|
|
||||||
name, value, inline := strings.Cut(arg[2:], "=")
|
|
||||||
def := flagByLong(name)
|
|
||||||
if def == nil {
|
|
||||||
return nil, fmt.Errorf("unknown flag --%s", name)
|
|
||||||
}
|
|
||||||
return []flagArg{{def, value, inline}}, nil
|
|
||||||
}
|
|
||||||
if name, _, _ := strings.Cut(arg[1:], "="); len(name) > 1 && flagByLong(name) != nil {
|
|
||||||
return nil, fmt.Errorf("unknown flag %s; use --%s", arg, name)
|
|
||||||
}
|
|
||||||
var flags []flagArg
|
|
||||||
letters := arg[1:]
|
|
||||||
for i, r := range letters {
|
|
||||||
letter := string(r)
|
|
||||||
def := flagByShort(letter)
|
|
||||||
if def == nil {
|
|
||||||
return nil, fmt.Errorf("unknown flag -%s", letter)
|
|
||||||
}
|
|
||||||
if !def.value {
|
|
||||||
flags = append(flags, flagArg{def: def})
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
rest := letters[i+len(letter):]
|
|
||||||
if strings.HasPrefix(rest, "=") {
|
|
||||||
return nil, fmt.Errorf("-%s takes its value attached (-%s%s) or next (-%s %s); = belongs to --%s=%s",
|
|
||||||
letter, letter, rest[1:], letter, rest[1:], def.long, rest[1:])
|
|
||||||
}
|
|
||||||
return append(flags, flagArg{def, rest, rest != ""}), nil
|
|
||||||
}
|
|
||||||
return flags, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseArgs(argv []string) (invocation, error) {
|
|
||||||
in := invocation{repeat: 1, separator: "\n", format: "text"}
|
|
||||||
for i := 0; i < len(argv); i++ {
|
|
||||||
arg := argv[i]
|
|
||||||
if arg == "--" {
|
|
||||||
in.paths = append(in.paths, argv[i+1:]...)
|
|
||||||
return in, nil
|
|
||||||
}
|
|
||||||
if len(arg) < 2 || arg[0] != '-' {
|
|
||||||
in.paths = append(in.paths, arg)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
flags, err := splitFlags(arg)
|
|
||||||
if err != nil {
|
|
||||||
return in, err
|
|
||||||
}
|
|
||||||
for _, fl := range flags {
|
|
||||||
if !fl.def.value {
|
|
||||||
if fl.inline {
|
|
||||||
return in, fmt.Errorf("--%s takes no value", fl.def.long)
|
|
||||||
}
|
|
||||||
_ = fl.def.set(&in, "")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
value := fl.value
|
|
||||||
if !fl.inline {
|
|
||||||
if i++; i >= len(argv) {
|
|
||||||
return in, fmt.Errorf("--%s needs a value", fl.def.long)
|
|
||||||
}
|
|
||||||
value = argv[i]
|
|
||||||
}
|
|
||||||
if err := fl.def.set(&in, value); err != nil {
|
|
||||||
return in, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return in, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordFormat is one way to write a record out: the line each record renders,
|
|
||||||
// the header that precedes the first one, and the open/close frame plus the
|
|
||||||
// between-record separator a document form needs.
|
|
||||||
type recordFormat struct {
|
|
||||||
header func(*fejkdata.Record) string
|
|
||||||
line func(r *fejkdata.Record, table string) string
|
|
||||||
open string
|
|
||||||
close string
|
|
||||||
sep string
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordFormats is every --format that writes records. json frames the records
|
|
||||||
// as one array document; ndjson is the same column, one object per line.
|
|
||||||
var recordFormats = map[string]recordFormat{
|
|
||||||
"csv": {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }, sep: "\n"},
|
|
||||||
"json": {line: jsonLine, open: "[", close: "]", sep: ",\n"},
|
|
||||||
"ndjson": {line: jsonLine, sep: "\n"},
|
|
||||||
"sql": {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }, sep: "\n"},
|
|
||||||
}
|
|
||||||
|
|
||||||
func jsonLine(r *fejkdata.Record, _ string) string { return r.JSON() }
|
|
||||||
|
|
||||||
// writesRecords reports whether the format writes records rather than plain text.
|
|
||||||
func (in invocation) writesRecords() bool {
|
|
||||||
return in.format != "text"
|
|
||||||
}
|
|
||||||
|
|
||||||
// formatNames lists the --format values, text included, for the misuse error.
|
|
||||||
func formatNames() string {
|
|
||||||
names := []string{"text"}
|
|
||||||
for name := range recordFormats {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
return strings.Join(names[:len(names)-1], ", ") + " or " + names[len(names)-1]
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkFlags rejects a flag value or combination that cannot run.
|
|
||||||
func (in invocation) checkFlags() error {
|
|
||||||
if _, ok := recordFormats[in.format]; !ok && in.format != "text" {
|
|
||||||
return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format)
|
|
||||||
}
|
|
||||||
if in.tableSet && in.table == "" {
|
|
||||||
return errors.New("--table names the INSERT target, so it cannot be empty")
|
|
||||||
}
|
|
||||||
if in.tableSet && in.format != "sql" {
|
|
||||||
return errors.New("--table names the INSERT target, so it needs --format sql")
|
|
||||||
}
|
|
||||||
if in.writesRecords() && in.separatorSet {
|
|
||||||
return errors.New("--separator joins text values, so it has no effect with --format " + in.format)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// check rejects a flag combination or an argument that cannot run, and reports
|
|
||||||
// what the argument names, so its shape is settled before any data is read.
|
|
||||||
func (in invocation) check() (argKind, error) {
|
|
||||||
if err := in.checkFlags(); err != nil {
|
|
||||||
return argPath, err
|
|
||||||
}
|
|
||||||
if in.list && len(in.paths) > 0 {
|
|
||||||
return argPath, errors.New("--list takes no path")
|
|
||||||
}
|
|
||||||
if in.list && (in.repeatSet || in.separatorSet || in.formatSet || in.tableSet) {
|
|
||||||
return argPath, errors.New("--list takes no --repeat, --separator, --format or --table")
|
|
||||||
}
|
|
||||||
if in.list {
|
|
||||||
return argPath, nil
|
|
||||||
}
|
|
||||||
if len(in.paths) != 1 {
|
|
||||||
return argPath, fmt.Errorf("expected one path or template, got %d%s", len(in.paths), templateSplit(in.paths))
|
|
||||||
}
|
|
||||||
return classify(in.paths[0])
|
|
||||||
}
|
|
||||||
|
|
||||||
func (in invocation) options() []fejkdata.Option {
|
|
||||||
var opts []fejkdata.Option
|
|
||||||
if in.noShipped {
|
|
||||||
opts = append(opts, fejkdata.WithoutShippedData())
|
|
||||||
}
|
|
||||||
for _, dir := range in.dirs {
|
|
||||||
opts = append(opts, fejkdata.WithDataPath(dir))
|
|
||||||
}
|
|
||||||
if in.seeded {
|
|
||||||
opts = append(opts, fejkdata.WithSeed(in.seed))
|
|
||||||
}
|
|
||||||
return opts
|
|
||||||
}
|
|
||||||
|
|
||||||
// write streams the argument's renders to w, repeat of them joined by the
|
|
||||||
// separator and ended by a newline. A value that renders once renders every time,
|
|
||||||
// so a render failure comes before anything is written; a write failure surfaces
|
|
||||||
// from Flush, bufio keeping the first one.
|
|
||||||
func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) error {
|
|
||||||
if in.writesRecords() {
|
|
||||||
return in.writeRecords(f, kind, w)
|
|
||||||
}
|
|
||||||
draw, err := in.textDraw(f, kind, in.paths[0])
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
out := bufio.NewWriter(w)
|
|
||||||
for i := 0; i < in.repeat; i++ {
|
|
||||||
v, err := draw()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if i > 0 {
|
|
||||||
out.WriteString(in.separator)
|
|
||||||
}
|
|
||||||
out.WriteString(v)
|
|
||||||
}
|
|
||||||
out.WriteString("\n")
|
|
||||||
return out.Flush()
|
|
||||||
}
|
|
||||||
|
|
||||||
// textDraw builds what one text render yields: the value, from a path or an
|
|
||||||
// inline template.
|
|
||||||
func (in invocation) textDraw(f *fejkdata.Generator, kind argKind, arg string) (func() (string, error), error) {
|
|
||||||
if kind != argTemplate {
|
|
||||||
return func() (string, error) { return f.Fake(arg) }, nil
|
|
||||||
}
|
|
||||||
t, err := f.NewTemplate(arg)
|
|
||||||
if err != nil {
|
|
||||||
return nil, templateError{err}
|
|
||||||
}
|
|
||||||
return func() (string, error) { return t.Fake(), nil }, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordStream builds the record drawer for the argument, plus the INSERT table
|
|
||||||
// a sql format names.
|
|
||||||
func (in invocation) recordStream(f *fejkdata.Generator, kind argKind, arg string) (func() (*fejkdata.Record, error), string, error) {
|
|
||||||
record := func() (*fejkdata.Record, error) { return f.FakeRecord(arg) }
|
|
||||||
if kind == argTemplate {
|
|
||||||
t, err := f.NewRecordTemplate(arg)
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", templateError{err}
|
|
||||||
}
|
|
||||||
record = func() (*fejkdata.Record, error) { return t.Fake(), nil }
|
|
||||||
}
|
|
||||||
table := in.table
|
|
||||||
if table == "" {
|
|
||||||
table = defaultTable(arg, kind)
|
|
||||||
}
|
|
||||||
return record, table, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeRecords streams a record per line in the chosen format, framing a document
|
|
||||||
// form with its open/close brackets and a header preceding the first record.
|
|
||||||
func (in invocation) writeRecords(f *fejkdata.Generator, kind argKind, w io.Writer) error {
|
|
||||||
record, table, err := in.recordStream(f, kind, in.paths[0])
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
format := recordFormats[in.format]
|
|
||||||
out := bufio.NewWriter(w)
|
|
||||||
if format.open != "" {
|
|
||||||
out.WriteString(format.open)
|
|
||||||
out.WriteByte('\n')
|
|
||||||
}
|
|
||||||
for i := 0; i < in.repeat; i++ {
|
|
||||||
r, err := record()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if i == 0 && format.header != nil {
|
|
||||||
out.WriteString(format.header(r))
|
|
||||||
out.WriteByte('\n')
|
|
||||||
}
|
|
||||||
if i > 0 {
|
|
||||||
out.WriteString(format.sep)
|
|
||||||
}
|
|
||||||
out.WriteString(format.line(r, table))
|
|
||||||
}
|
|
||||||
if format.close != "" {
|
|
||||||
out.WriteByte('\n')
|
|
||||||
out.WriteString(format.close)
|
|
||||||
}
|
|
||||||
out.WriteByte('\n')
|
|
||||||
return out.Flush()
|
|
||||||
}
|
|
||||||
|
|
||||||
// defaultTable names the INSERT target when --table is absent: the path's last
|
|
||||||
// segment, or "records" for an inline template that sits in no folder.
|
|
||||||
func defaultTable(arg string, kind argKind) string {
|
|
||||||
if kind == argTemplate {
|
|
||||||
return "records"
|
|
||||||
}
|
|
||||||
var names strings.Builder // the path with its [selectors] cut out
|
|
||||||
for depth, i := 0, 0; i < len(arg); i++ {
|
|
||||||
switch {
|
|
||||||
case arg[i] == '[':
|
|
||||||
depth++
|
|
||||||
case arg[i] == ']':
|
|
||||||
depth--
|
|
||||||
case depth == 0:
|
|
||||||
names.WriteByte(arg[i])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
segments := strings.Split(names.String(), ".")
|
|
||||||
return segments[len(segments)-1]
|
|
||||||
}
|
|
||||||
|
|
||||||
// templateError marks a render failure that is the argument's own fault — an
|
|
||||||
// inline template that does not compile. run reports it as misuse (exit 2, with a
|
|
||||||
// pointer to --help), unlike an unknown path, which is a runtime error (exit 1).
|
|
||||||
type templateError struct{ error }
|
|
||||||
|
|
||||||
func (e templateError) Unwrap() error { return e.error }
|
|
||||||
|
|
||||||
type argKind int
|
|
||||||
|
|
||||||
const (
|
|
||||||
argPath argKind = iota
|
|
||||||
argTemplate
|
|
||||||
)
|
|
||||||
|
|
||||||
// classify reads what a positional argument names by its shape (see fejkdata.IsTemplate).
|
|
||||||
func classify(arg string) (argKind, error) {
|
|
||||||
inline, err := fejkdata.IsTemplate(arg)
|
|
||||||
if inline {
|
|
||||||
return argTemplate, nil
|
|
||||||
}
|
|
||||||
return argPath, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) }
|
|
||||||
|
|
||||||
// run returns the exit code: 0 ok, 1 runtime error, 2 misuse.
|
|
||||||
func run(args []string, stdout, stderr io.Writer) int {
|
|
||||||
in, err := parseArgs(args)
|
|
||||||
if err != nil {
|
|
||||||
return misuse(stderr, err)
|
|
||||||
}
|
|
||||||
if in.help {
|
|
||||||
fmt.Fprint(stdout, usage)
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
if in.version {
|
|
||||||
fmt.Fprintln(stdout, "fejkdata "+buildVersion())
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
kind, err := in.check()
|
|
||||||
if err != nil {
|
|
||||||
return misuse(stderr, err)
|
|
||||||
}
|
|
||||||
f, err := fejkdata.New(in.options()...)
|
|
||||||
if errors.Is(err, fejkdata.ErrNoData) {
|
|
||||||
return misuse(stderr, errors.New("--no-shipped-data needs at least one --data-path"))
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintln(stderr, err)
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
if in.list {
|
|
||||||
for _, p := range f.List() {
|
|
||||||
fmt.Fprintln(stdout, p)
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
if err := in.write(f, kind, stdout); err != nil {
|
|
||||||
var te templateError
|
|
||||||
if errors.As(err, &te) {
|
|
||||||
return misuse(stderr, te.error)
|
|
||||||
}
|
|
||||||
fmt.Fprintln(stderr, err)
|
|
||||||
if errors.Is(err, fejkdata.ErrNoColumns) {
|
|
||||||
fmt.Fprintln(stderr, "wrap that JSON in single quotes, which keep a shell from expanding its braces")
|
|
||||||
}
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// templateSplit names the cure for a template a shell split on its spaces, which is
|
|
||||||
// what several arguments carrying a token mean.
|
|
||||||
func templateSplit(paths []string) string {
|
|
||||||
for _, p := range paths {
|
|
||||||
if strings.Contains(p, "{") {
|
|
||||||
return `; a template's spaces split the argument, so wrap the whole argument in "…", or in '…' where it carries double quotes`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func misuse(stderr io.Writer, err error) int {
|
|
||||||
// A library error already names the program, so the prefix is not doubled.
|
|
||||||
fmt.Fprintf(stderr, "fejkdata: %s\ntry 'fejkdata --help'\n", strings.TrimPrefix(err.Error(), "fejkdata: "))
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildVersion is the module version go install stamps into the binary, or "devel".
|
|
||||||
func buildVersion() string {
|
|
||||||
if info, ok := debug.ReadBuildInfo(); ok && info.Main.Version != "" {
|
|
||||||
return info.Main.Version
|
|
||||||
}
|
|
||||||
return "devel"
|
|
||||||
}
|
|
||||||
|
|||||||
+30
-624
@@ -2,640 +2,46 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/csv"
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"os/exec"
|
||||||
"regexp"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
func TestMain(m *testing.M) {
|
||||||
svSE = "../../data/sv_SE"
|
if os.Getenv("FEJKDATA_RUN_MAIN") == "1" {
|
||||||
enUS = "../../data/en_US"
|
main()
|
||||||
misc = "../../data/misc"
|
}
|
||||||
)
|
os.Exit(m.Run())
|
||||||
|
|
||||||
func runOut(args ...string) (int, string, string) {
|
|
||||||
var out, errb bytes.Buffer
|
|
||||||
code := run(args, &out, &errb)
|
|
||||||
return code, out.String(), errb.String()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunOutputsValue(t *testing.T) {
|
func TestMainRendersTheShippedData(t *testing.T) {
|
||||||
code, out, errb := runOut("--data-path", svSE, "person")
|
cmd := exec.Command(os.Args[0], "--seed", "1")
|
||||||
if code != 0 {
|
cmd.Env = append(os.Environ(), "FEJKDATA_RUN_MAIN=1")
|
||||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
cmd.Stdin = strings.NewReader("{/misc.territory[SE].alpha2} {/en_US.person.last} {/sv_SE.address.locality}")
|
||||||
}
|
var stderr bytes.Buffer
|
||||||
if strings.TrimSpace(out) == "" {
|
cmd.Stderr = &stderr
|
||||||
t.Fatalf("empty output, stderr=%q", errb)
|
out, err := cmd.Output()
|
||||||
}
|
if fields := strings.Fields(string(out)); err != nil || len(fields) < 3 || fields[0] != "SE" {
|
||||||
if !strings.HasSuffix(out, "\n") {
|
t.Errorf("fejkdata = %v, %q, %q, want SE followed by a last name and a locality", err, out, stderr.String())
|
||||||
t.Errorf("output should end with newline, got %q", out)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunDotPath(t *testing.T) {
|
func TestMainWithoutASeedDrawsANewSeedEachRun(t *testing.T) {
|
||||||
code, full, _ := runOut("--seed", "7", "--data-path", svSE, "person")
|
var outs [2]string
|
||||||
if code != 0 {
|
for i := range outs {
|
||||||
t.Fatalf("person run = %d", code)
|
cmd := exec.Command(os.Args[0])
|
||||||
}
|
cmd.Env = append(os.Environ(), "FEJKDATA_RUN_MAIN=1")
|
||||||
code, last, errb := runOut("--seed", "7", "--data-path", svSE, "person.last")
|
cmd.Stdin = strings.NewReader("{uuid()}")
|
||||||
if code != 0 {
|
var stderr bytes.Buffer
|
||||||
t.Fatalf("person.last run = %d, stderr=%q", code, errb)
|
cmd.Stderr = &stderr
|
||||||
}
|
out, err := cmd.Output()
|
||||||
if strings.TrimSpace(last) == "" {
|
if err != nil || len(out) == 0 {
|
||||||
t.Fatal("empty person.last output")
|
t.Fatalf("fejkdata without --seed = %v, %q, %q", err, out, stderr.String())
|
||||||
}
|
|
||||||
if last == full {
|
|
||||||
t.Errorf("person.last %q should differ from person %q", last, full)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunSeedSpellings(t *testing.T) {
|
|
||||||
_, want, _ := runOut("--seed", "42", "--data-path", svSE, "address")
|
|
||||||
for _, args := range [][]string{
|
|
||||||
{"--seed=42", "--data-path", svSE, "address"},
|
|
||||||
{"-s", "42", "-d", svSE, "address"},
|
|
||||||
{"--data-path", svSE, "address", "--seed", "42"},
|
|
||||||
} {
|
|
||||||
code, got, errb := runOut(args...)
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
|
|
||||||
}
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("run(%v) = %q, want %q", args, got, want)
|
|
||||||
}
|
}
|
||||||
}
|
outs[i] = string(out)
|
||||||
}
|
}
|
||||||
|
if outs[0] == outs[1] {
|
||||||
func TestRunShortFlagValues(t *testing.T) {
|
t.Errorf("two runs without --seed both printed %q, want a new seed each run", outs[0])
|
||||||
_, want, _ := runOut("--seed", "42", "--data-path", svSE, "address")
|
|
||||||
for _, args := range [][]string{
|
|
||||||
{"-s42", "-d", svSE, "address"},
|
|
||||||
{"-s", "42", "-d" + svSE, "address"},
|
|
||||||
{"-d", svSE, "address", "-s42"},
|
|
||||||
} {
|
|
||||||
code, got, errb := runOut(args...)
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("run(%v) = %d, stderr=%q", args, code, errb)
|
|
||||||
}
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("run(%v) = %q, want %q", args, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_, three, _ := runOut("-s", "1", "-n3", "-d", svSE, "word")
|
|
||||||
if lines := strings.Split(strings.TrimRight(three, "\n"), "\n"); len(lines) != 3 {
|
|
||||||
t.Errorf("-n3 gave %d lines: %q", len(lines), three)
|
|
||||||
}
|
|
||||||
for _, c := range []struct {
|
|
||||||
args []string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{[]string{"-s=42", "-d", svSE, "address"}, "-s42"},
|
|
||||||
{[]string{"-s=42", "-d", svSE, "address"}, "--seed=42"},
|
|
||||||
{[]string{"-d=" + svSE, "address"}, "--data-path="},
|
|
||||||
{[]string{"-nd", "3", "-d", svSE, "word"}, `--repeat needs an integer in 1..1048576, got "d"`},
|
|
||||||
{[]string{"--seed=", "-d", svSE, "word"}, `--seed needs an unsigned integer, got ""`},
|
|
||||||
} {
|
|
||||||
code, out, errb := runOut(c.args...)
|
|
||||||
if code != 2 || out != "" {
|
|
||||||
t.Errorf("run(%v) = %d, stdout %q, want misuse", c.args, code, out)
|
|
||||||
}
|
|
||||||
if !strings.Contains(errb, c.want) {
|
|
||||||
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
code, out, _ := runOut("-hd", svSE)
|
|
||||||
if code != 0 || !strings.Contains(out, "Usage") {
|
|
||||||
t.Errorf("-hd (bundled help) = %d, %q, want usage", code, out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunDoubleDashEndsFlags(t *testing.T) {
|
|
||||||
code, out, errb := runOut("--data-path", svSE, "--", "person")
|
|
||||||
if code != 0 || strings.TrimSpace(out) == "" {
|
|
||||||
t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
|
|
||||||
}
|
|
||||||
code, _, errb = runOut("--data-path", svSE, "--", "--list")
|
|
||||||
if code != 1 || !strings.Contains(errb, "--list") {
|
|
||||||
t.Errorf("after --, --list should be a path: code %d, stderr %q", code, errb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunSingleDashLongIsRejected(t *testing.T) {
|
|
||||||
for _, c := range []struct {
|
|
||||||
args []string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{[]string{"-seed", "42", "-d", svSE, "person"}, "use --seed"},
|
|
||||||
{[]string{"-seed=42", "-d", svSE, "person"}, "use --seed"},
|
|
||||||
{[]string{"-data-path", svSE, "person"}, "use --data-path"},
|
|
||||||
{[]string{"-list", "-d", svSE}, "use --list"},
|
|
||||||
{[]string{"--nope", "-d", svSE, "person"}, "unknown flag --nope"},
|
|
||||||
{[]string{"-x", "-d", svSE, "person"}, "unknown flag -x"},
|
|
||||||
} {
|
|
||||||
code, _, errb := runOut(c.args...)
|
|
||||||
if code != 2 {
|
|
||||||
t.Errorf("run(%v) = %d, want 2", c.args, code)
|
|
||||||
}
|
|
||||||
if !strings.Contains(errb, c.want) {
|
|
||||||
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunFlagValues(t *testing.T) {
|
|
||||||
for _, c := range []struct {
|
|
||||||
args []string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{[]string{"--seed", "-d", svSE, "person"}, `--seed needs an unsigned integer, got "-d"`},
|
|
||||||
{[]string{"-d", svSE, "person", "--seed"}, "--seed needs a value"},
|
|
||||||
{[]string{"--seed", "x", "-d", svSE, "person"}, "--seed"},
|
|
||||||
{[]string{"--list=1", "-d", svSE}, "--list takes no value"},
|
|
||||||
{[]string{"--repeat", "0", "-d", svSE, "person"}, "--repeat"},
|
|
||||||
{[]string{"-n", "-1", "-d", svSE, "person"}, "--repeat"},
|
|
||||||
} {
|
|
||||||
code, _, errb := runOut(c.args...)
|
|
||||||
if code != 2 {
|
|
||||||
t.Errorf("run(%v) = %d, want 2", c.args, code)
|
|
||||||
}
|
|
||||||
if !strings.Contains(errb, c.want) {
|
|
||||||
t.Errorf("run(%v) stderr = %q, want %q", c.args, errb, c.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunRepeat(t *testing.T) {
|
|
||||||
code, out, errb := runOut("--seed", "1", "--repeat", "3", "--data-path", svSE, "word")
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
|
||||||
}
|
|
||||||
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
|
|
||||||
if len(lines) != 3 {
|
|
||||||
t.Errorf("repeat=3 gave %d lines: %q", len(lines), out)
|
|
||||||
}
|
|
||||||
_, short, _ := runOut("--seed", "1", "-n", "3", "-d", svSE, "word")
|
|
||||||
if short != out {
|
|
||||||
t.Errorf("-n 3 = %q, want the same as --repeat 3 %q", short, out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunSeparator(t *testing.T) {
|
|
||||||
code, out, errb := runOut("--repeat", "3", "--separator", ",", "--data-path", svSE, "word")
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
|
||||||
}
|
|
||||||
if n := strings.Count(out, "\n"); n != 1 {
|
|
||||||
t.Errorf("want one trailing newline, got %d: %q", n, out)
|
|
||||||
}
|
|
||||||
if !strings.Contains(out, ",") {
|
|
||||||
t.Errorf("values should be comma-joined: %q", out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunRepeatAdvancesRNG(t *testing.T) {
|
|
||||||
code, out, errb := runOut("--seed", "1", "--repeat", "5", "--data-path", svSE, "person")
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
|
||||||
}
|
|
||||||
uniq := map[string]bool{}
|
|
||||||
for _, l := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
|
|
||||||
uniq[l] = true
|
|
||||||
}
|
|
||||||
if len(uniq) < 2 {
|
|
||||||
t.Errorf("repeat should vary output, all identical: %q", out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunMisuse(t *testing.T) {
|
|
||||||
for _, args := range [][]string{{}, {"--data-path", svSE}, {"-d", svSE, "person", "word"}, {"-d", svSE, "--list", "person"}, {"--no-shipped-data", "sv_SE.person"}} {
|
|
||||||
code, out, errb := runOut(args...)
|
|
||||||
if code != 2 {
|
|
||||||
t.Errorf("run(%v) = %d, want 2", args, code)
|
|
||||||
}
|
|
||||||
if !strings.Contains(errb, "try 'fejkdata --help'") {
|
|
||||||
t.Errorf("run(%v) stderr = %q, want a pointer to --help", args, errb)
|
|
||||||
}
|
|
||||||
if out != "" {
|
|
||||||
t.Errorf("run(%v) stdout = %q, want nothing", args, out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// A template a shell split names both quotings, since which one cures it turns
|
|
||||||
// on whether the argument carries a double-quoted JSON.
|
|
||||||
_, _, errb := runOut("{date(1990-01-01,2010-12-31,", "January", "2,", "2006)}")
|
|
||||||
for _, want := range []string{`wrap the whole argument in "…"`, `or in '…'`} {
|
|
||||||
if !strings.Contains(errb, want) {
|
|
||||||
t.Errorf("a split template stderr = %q, want it to name %s", errb, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunMultipleDataPaths(t *testing.T) {
|
|
||||||
code, out, errb := runOut("-d", enUS, "--data-path", svSE, "person")
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("run(multi-dir) = %d, stderr=%q", code, errb)
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(out) == "" {
|
|
||||||
t.Fatal("empty output for multi-dir run")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunUnknownCategoryFails(t *testing.T) {
|
|
||||||
code, _, errb := runOut("--data-path", svSE, "nope")
|
|
||||||
if code != 1 {
|
|
||||||
t.Fatalf("run = %d, want 1", code)
|
|
||||||
}
|
|
||||||
if !strings.Contains(errb, "nope") {
|
|
||||||
t.Errorf("stderr %q should name the unknown category", errb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunList(t *testing.T) {
|
|
||||||
code, out, errb := runOut("--data-path", svSE, "--list")
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"person", "person.last", "address"} {
|
|
||||||
if !strings.Contains(out, want) {
|
|
||||||
t.Errorf("list output missing %q:\n%s", want, out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunHelpOnStdout(t *testing.T) {
|
|
||||||
for _, h := range []string{"-h", "--help"} {
|
|
||||||
code, out, errb := runOut(h)
|
|
||||||
if code != 0 {
|
|
||||||
t.Errorf("%s = %d, want 0", h, code)
|
|
||||||
}
|
|
||||||
if !strings.Contains(out, "Usage") || errb != "" {
|
|
||||||
t.Errorf("%s: stdout = %q, stderr = %q, want usage on stdout only", h, out, errb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunVersion(t *testing.T) {
|
|
||||||
code, out, errb := runOut("--version")
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("--version = %d, stderr=%q", code, errb)
|
|
||||||
}
|
|
||||||
version, ok := strings.CutPrefix(out, "fejkdata ")
|
|
||||||
if !ok || strings.TrimSpace(version) == "" {
|
|
||||||
t.Errorf("--version = %q, want the command name and a version on stdout", out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunMissingDirFails(t *testing.T) {
|
|
||||||
code, _, errb := runOut("--data-path", "../../data/nope", "person")
|
|
||||||
if code != 1 {
|
|
||||||
t.Fatalf("run = %d, want 1", code)
|
|
||||||
}
|
|
||||||
if errb == "" {
|
|
||||||
t.Error("want an error message on stderr")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunShippedDataByDefault(t *testing.T) {
|
|
||||||
code, out, errb := runOut("--seed", "1", "sv_SE.person")
|
|
||||||
if code != 0 || strings.TrimSpace(out) == "" {
|
|
||||||
t.Fatalf("run = %d, out=%q, stderr=%q", code, out, errb)
|
|
||||||
}
|
|
||||||
code, list, _ := runOut("--list")
|
|
||||||
if code != 0 || !strings.Contains(list, "en_US.person\n") || !strings.Contains(list, "misc.uuid\n") {
|
|
||||||
t.Errorf("--list without --data-path = %d, %q", code, list)
|
|
||||||
}
|
|
||||||
code, _, errb = runOut("person")
|
|
||||||
if code != 1 || !strings.Contains(errb, "person") {
|
|
||||||
t.Errorf("a category outside the shipped tree: code %d, stderr %q", code, errb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestClassify(t *testing.T) {
|
|
||||||
for arg, want := range map[string]argKind{"sv_SE.person": argPath, "name: {x}": argTemplate} {
|
|
||||||
if got, err := classify(arg); err != nil || got != want {
|
|
||||||
t.Errorf("classify(%q) = %v, %v; want %v", arg, got, err, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, err := classify("[abc]"); err == nil || !strings.Contains(err.Error(), `"["`) || !strings.Contains(err.Error(), "JSON") {
|
|
||||||
t.Errorf("classify([abc]) = %v; want it rejected naming the leading bracket", err)
|
|
||||||
}
|
|
||||||
if got, err := classify("geo.SE.municipality[St. Louis].name"); err != nil || got != argPath {
|
|
||||||
t.Errorf("classify(a selecting path) = %v, %v; want a path", got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunSelectsATableRow(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
for name, content := range map[string]string{
|
|
||||||
"region.json": `{"format":"{name}","rows":"region.tsv","key":"code","name":"name"}`,
|
|
||||||
"region.tsv": "code\tname\n01\tStockholms län\n12\tSkåne län\n",
|
|
||||||
} {
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if code, out, errb := runOut("--no-shipped-data", "-d", dir, "region[12]"); code != 0 || out != "Skåne län\n" {
|
|
||||||
t.Fatalf("region[12] = %d, %q, stderr=%q", code, out, errb)
|
|
||||||
}
|
|
||||||
if code, out, _ := runOut("--no-shipped-data", "-d", dir, "region[Skåne län].code"); code != 0 || out != "12\n" {
|
|
||||||
t.Fatalf("region[Skåne län].code = %d, %q", code, out)
|
|
||||||
}
|
|
||||||
if code, out, _ := runOut("--no-shipped-data", "-d", dir, "--format", "sql", "region[12]"); code != 0 || out != `INSERT INTO "region" ("code", "name") VALUES ('12', 'Skåne län');`+"\n" {
|
|
||||||
t.Fatalf("--format sql region[12] = %d, %q, want the table named without its selector", code, out)
|
|
||||||
}
|
|
||||||
if code, _, errb := runOut("--no-shipped-data", "-d", dir, "region[99]"); code != 1 || !strings.Contains(errb, `"99"`) {
|
|
||||||
t.Fatalf("region[99] = %d, stderr=%q, want a runtime error naming the row", code, errb)
|
|
||||||
}
|
|
||||||
if code, _, errb := runOut("--no-shipped-data", "-d", dir, "[Skåne län]"); code != 2 || !strings.Contains(errb, "JSON") {
|
|
||||||
t.Fatalf("[Skåne län] = %d, stderr=%q, want misuse: a leading bracket that is no JSON names nothing", code, errb)
|
|
||||||
}
|
|
||||||
if code, out, _ := runOut("--seed", "1", "--format", "csv", "misc.territory[SE]"); code != 0 || !strings.HasPrefix(out, "alpha2,") || !strings.Contains(out, "\nSE,SWE,") {
|
|
||||||
t.Fatalf("--format csv misc.territory[SE] = %d, %q", code, out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUsageReferencesResolve(t *testing.T) {
|
|
||||||
for _, token := range regexp.MustCompile(`\{/[^}]+\}`).FindAllString(usage, -1) {
|
|
||||||
path := token[2 : len(token)-1]
|
|
||||||
code, out, errb := runOut("--seed", "1", path)
|
|
||||||
if code != 0 || strings.TrimSpace(out) == "" {
|
|
||||||
t.Errorf("usage advertises %s: run %s = %d, %q, stderr %q", token, path, code, out, errb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunShapeMisuseBeforeLoad(t *testing.T) {
|
|
||||||
code, _, errb := runOut("--no-shipped-data", "[abc]")
|
|
||||||
if code != 2 || !strings.Contains(errb, "[abc]") || strings.Contains(errb, "--data-path") {
|
|
||||||
t.Fatalf("shape misuse with no data = %d, %q; want the shape error before any load", code, errb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunInlineTemplate(t *testing.T) {
|
|
||||||
code, out, errb := runOut("--seed", "1", "name: {/sv_SE.person.last}")
|
|
||||||
if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
|
|
||||||
t.Fatalf("inline format string = %d, %q, stderr %q", code, out, errb)
|
|
||||||
}
|
|
||||||
code, out, errb = runOut("--seed", "1", `{"format":"name: {x}","x":["bosse","lina"]}`)
|
|
||||||
if code != 0 || (out != "name: bosse\n" && out != "name: lina\n") {
|
|
||||||
t.Fatalf("inline JSON template = %d, %q, want one name, stderr %q", code, out, errb)
|
|
||||||
}
|
|
||||||
code, out, errb = runOut("--seed", "1", `"name: {/sv_SE.person.last}"`)
|
|
||||||
if code != 0 || !strings.HasPrefix(out, "name: ") || strings.Contains(out, "{") {
|
|
||||||
t.Fatalf("inline JSON string = %d, %q, stderr %q", code, out, errb)
|
|
||||||
}
|
|
||||||
code, out, errb = runOut("--seed", "1", "-n", "2", `{digits(1)}`)
|
|
||||||
if code != 0 || len(strings.Split(strings.TrimRight(out, "\n"), "\n")) != 2 {
|
|
||||||
t.Fatalf("inline template with --repeat = %d, %q, stderr %q", code, out, errb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunTemplateMisuse(t *testing.T) {
|
|
||||||
for arg, want := range map[string]string{
|
|
||||||
"{bad": "unterminated",
|
|
||||||
"[red,green]": `starts with "["`,
|
|
||||||
`{"format":"x"}`: "is a string",
|
|
||||||
"name: {/no.such.path}": "no entry",
|
|
||||||
"{/sv_SE.person}": "write sv_SE.person",
|
|
||||||
"x[1]y": `"]"`,
|
|
||||||
` ["a","b"] `: "may not be padded",
|
|
||||||
} {
|
|
||||||
code, out, errb := runOut("--seed", "1", arg)
|
|
||||||
if code != 2 || out != "" || !strings.Contains(errb, "try 'fejkdata --help'") || !strings.Contains(errb, want) {
|
|
||||||
t.Errorf("run(%q) = %d, %q, %q; want misuse naming %q and --help", arg, code, out, errb, want)
|
|
||||||
}
|
|
||||||
if strings.Contains(errb, "fejkdata: fejkdata:") {
|
|
||||||
t.Errorf("run(%q) doubled the program prefix: %q", arg, errb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunNoShippedData(t *testing.T) {
|
|
||||||
code, list, errb := runOut("--no-shipped-data", "-d", misc, "--list")
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
|
||||||
}
|
|
||||||
if strings.Contains(list, "sv_SE") || !strings.Contains(list, "uuid\n") {
|
|
||||||
t.Errorf("--no-shipped-data --list = %q, want only the given dir", list)
|
|
||||||
}
|
|
||||||
code, out, _ := runOut("--no-shipped-data", "-d", misc, "-s", "3", "uuid")
|
|
||||||
if code != 0 || strings.TrimSpace(out) == "" {
|
|
||||||
t.Errorf("run = %d, out=%q", code, out)
|
|
||||||
}
|
|
||||||
code, _, errb = runOut("--no-shipped-data", "sv_SE.person")
|
|
||||||
if code != 2 || !strings.Contains(errb, "--no-shipped-data needs at least one --data-path") {
|
|
||||||
t.Errorf("--no-shipped-data alone = %d, %q, want misuse naming --data-path", code, errb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunRepeatIsBounded(t *testing.T) {
|
|
||||||
for _, args := range [][]string{
|
|
||||||
{"--repeat", "9223372036854775807", "sv_SE.word"},
|
|
||||||
{"--repeat", "1048577", "sv_SE.word"},
|
|
||||||
{"-n", "99999999999", "sv_SE.word"},
|
|
||||||
} {
|
|
||||||
code, out, errb := runOut(args...)
|
|
||||||
if code != 2 || out != "" || !strings.Contains(errb, "1..1048576") {
|
|
||||||
t.Errorf("run(%v) = %d, %q, %q, want misuse naming the range", args, code, out, errb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
dir := t.TempDir()
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte(`"x"`), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
code, out, errb := runOut("--no-shipped-data", "-d", dir, "--repeat", "1048576", "--separator", "", "x")
|
|
||||||
if code != 0 || len(out) != 1048576+1 {
|
|
||||||
t.Errorf("repeat at the cap = %d, %d bytes, stderr %q; want every render streamed", code, len(out), errb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunListTakesNoRepeatOrSeparator(t *testing.T) {
|
|
||||||
for _, args := range [][]string{{"--list", "-n", "3"}, {"--list", "--separator", ","}} {
|
|
||||||
code, _, errb := runOut(args...)
|
|
||||||
if code != 2 || !strings.Contains(errb, "--list takes no") {
|
|
||||||
t.Errorf("run(%v) = %d, %q, want misuse", args, code, errb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunUnknownFlagIsNamedByRune(t *testing.T) {
|
|
||||||
code, _, errb := runOut("-ä", "sv_SE.word")
|
|
||||||
if code != 2 || !strings.Contains(errb, "unknown flag -ä") {
|
|
||||||
t.Errorf("run(-ä) = %d, %q, want the flag named whole", code, errb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunEmptyDataPathIsNamed(t *testing.T) {
|
|
||||||
code, _, errb := runOut("--data-path=", "sv_SE.word")
|
|
||||||
if code != 1 || !strings.Contains(errb, "empty") {
|
|
||||||
t.Errorf("run(--data-path=) = %d, %q, want the empty path named", code, errb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func recordDir(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
dir := t.TempDir()
|
|
||||||
content := `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"]}`
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "users.json"), []byte(content), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return dir
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunRecordJSON(t *testing.T) {
|
|
||||||
code, out, errb := runOut("--seed", "1", "--format", "json", "--repeat", "2", "--data-path", recordDir(t), "users")
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
|
||||||
}
|
|
||||||
var arr []map[string]string
|
|
||||||
if err := json.Unmarshal([]byte(out), &arr); err != nil {
|
|
||||||
t.Fatalf("json output is not one JSON array: %v\n%q", err, out)
|
|
||||||
}
|
|
||||||
if len(arr) != 2 || len(arr[0]) != 2 || arr[0]["first"] == "" || arr[0]["last"] == "" {
|
|
||||||
t.Fatalf("json output = %q, want an array of two records with first and last columns", out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunRecordNDJSON(t *testing.T) {
|
|
||||||
code, out, errb := runOut("--seed", "1", "--format", "ndjson", "--repeat", "2", "--data-path", recordDir(t), "users")
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
|
||||||
}
|
|
||||||
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
|
|
||||||
if len(lines) != 2 {
|
|
||||||
t.Fatalf("ndjson output = %d lines %q, want one object per line", len(lines), out)
|
|
||||||
}
|
|
||||||
for _, line := range lines {
|
|
||||||
var m map[string]string
|
|
||||||
if err := json.Unmarshal([]byte(line), &m); err != nil || len(m) != 2 {
|
|
||||||
t.Fatalf("ndjson line %q is not one object: %v", line, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunRecordCSVRoundTrips(t *testing.T) {
|
|
||||||
code, out, errb := runOut("--seed", "1", "--format", "csv", "--repeat", "3", "--data-path", recordDir(t), "users")
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
|
||||||
}
|
|
||||||
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
|
|
||||||
if len(lines) != 4 {
|
|
||||||
t.Fatalf("csv output = %d lines %q, want a header and 3 rows", len(lines), out)
|
|
||||||
}
|
|
||||||
if lines[0] != "first,last" {
|
|
||||||
t.Fatalf("csv header = %q, want first,last", lines[0])
|
|
||||||
}
|
|
||||||
r, err := csv.NewReader(strings.NewReader(out)).ReadAll()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("csv output is not valid CSV: %v", err)
|
|
||||||
}
|
|
||||||
if len(r) != 4 || len(r[0]) != 2 {
|
|
||||||
t.Fatalf("csv parsed to %v, want 4 records of 2 fields", r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunRecordSQL(t *testing.T) {
|
|
||||||
code, out, errb := runOut("--seed", "1", "--format", "sql", "--repeat", "2", "--table", "people", "--data-path", recordDir(t), "users")
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
|
||||||
}
|
|
||||||
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
|
|
||||||
if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) {
|
|
||||||
t.Fatalf("sql output = %q, want two INSERT INTO people statements", out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunRecordDefaultTable(t *testing.T) {
|
|
||||||
code, out, errb := runOut("--seed", "1", "--format", "sql", "--data-path", recordDir(t), "users")
|
|
||||||
if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
|
|
||||||
t.Fatalf("default table = %d, %q, want the path's last segment; stderr %q", code, out, errb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunRecordInlineTemplate(t *testing.T) {
|
|
||||||
code, out, errb := runOut("--seed", "1", "--format", "json", `{"format":"{x}","x":["a","b"]}`)
|
|
||||||
if code != 0 {
|
|
||||||
t.Fatalf("inline record = %d, stderr=%q", code, errb)
|
|
||||||
}
|
|
||||||
var arr []map[string]string
|
|
||||||
if err := json.Unmarshal([]byte(out), &arr); err != nil || len(arr) != 1 || (arr[0]["x"] != "a" && arr[0]["x"] != "b") {
|
|
||||||
t.Fatalf("inline record json = %q, want one record with a column x (err %v)", out, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunRecordMisuse(t *testing.T) {
|
|
||||||
for _, c := range []struct {
|
|
||||||
args []string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{[]string{"--format", "yaml", "users"}, "--format takes csv, json, ndjson, sql or text"},
|
|
||||||
{[]string{"--format", "json", "--separator", ",", "users"}, "--separator joins text values"},
|
|
||||||
{[]string{"--table", "t", "users"}, "--table names the INSERT target"},
|
|
||||||
{[]string{"--format", "json", "--table", "t", "users"}, "--table names the INSERT target"},
|
|
||||||
{[]string{"--format", "sql", "--table", "", "users"}, "--table names the INSERT target"},
|
|
||||||
} {
|
|
||||||
code, out, errb := runOut(c.args...)
|
|
||||||
if code != 2 || out != "" || !strings.Contains(errb, c.want) {
|
|
||||||
t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunRecordOnABareValueIsRuntimeError(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "word.json"), []byte(`["a","b"]`), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
code, _, errb := runOut("--format", "json", "--data-path", dir, "word")
|
|
||||||
if code != 1 {
|
|
||||||
t.Fatalf("record on a choice = %d, want exit 1 (runtime error), stderr %q", code, errb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunRecordRejectsATopLevelRepeat(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
content := `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "rep.json"), []byte(content), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
code, _, errb := runOut("--format", "json", "--data-path", dir, "rep")
|
|
||||||
if code != 1 || !strings.Contains(errb, "carries repeat 3") {
|
|
||||||
t.Errorf("record on a repeating path = %d, %q; want exit 1 naming the repeat", code, errb)
|
|
||||||
}
|
|
||||||
code, _, errb = runOut("--format", "json", content)
|
|
||||||
if code != 2 || !strings.Contains(errb, "carries repeat 3") {
|
|
||||||
t.Errorf("record on a repeating inline template = %d, %q; want exit 2 naming the repeat", code, errb)
|
|
||||||
}
|
|
||||||
if code, out, _ := runOut("--seed", "1", "--data-path", dir, "rep"); code != 0 || strings.TrimSpace(out) != "x-|x-|x-" {
|
|
||||||
t.Errorf("text view = %d, %q; want the repeat still composed", code, out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunNumbersFollowTheShell(t *testing.T) {
|
|
||||||
_, want, _ := runOut("--seed", "7", "--repeat", "3", "sv_SE.word")
|
|
||||||
code, got, errb := runOut("--seed", "007", "--repeat", "+3", "sv_SE.word")
|
|
||||||
if code != 0 || got != want {
|
|
||||||
t.Errorf("run(--seed 007 --repeat +3) = %d, %q, stderr %q; want the same as --seed 7 --repeat 3 %q", code, got, errb, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunRecordOfAFieldlessPath pins the way out of asking a category of one value
|
|
||||||
// for columns: the record to write, and the quoting a shell needs to take it. The
|
|
||||||
// argument is well-formed and only the data cannot serve it, so the code is 1.
|
|
||||||
func TestRunRecordOfAFieldlessPath(t *testing.T) {
|
|
||||||
code, out, errb := runOut("--format", "csv", "sv_SE.personnummer")
|
|
||||||
if code != 1 {
|
|
||||||
t.Fatalf("run(--format csv sv_SE.personnummer) = %d, want 1", code)
|
|
||||||
}
|
|
||||||
for _, want := range []string{`{"format":"","personnummer":"{/sv_SE.personnummer}"}`, "single quotes"} {
|
|
||||||
if !strings.Contains(errb, want) {
|
|
||||||
t.Errorf("stderr = %q, want it to name %s", errb, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if out != "" {
|
|
||||||
t.Errorf("stdout = %q, want nothing", out)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+11
-20
@@ -1,21 +1,5 @@
|
|||||||
# Docker-based dev workflow — no local tooling beyond Docker is required.
|
|
||||||
#
|
|
||||||
# docker compose run --rm test
|
|
||||||
# docker compose run --rm vet
|
|
||||||
# docker compose run --rm build
|
|
||||||
#
|
|
||||||
# The full gate — vet, formatting and tests — is `docker build .`, which is what
|
|
||||||
# CI runs.
|
|
||||||
#
|
|
||||||
# Build caches persist in the `gocache` volume, so re-runs are fast and the
|
|
||||||
# host tree stays clean. Commands that rewrite source (fmt, tidy) keep your
|
|
||||||
# ownership when run with `--user "$(id -u):$(id -g)"`.
|
|
||||||
#
|
|
||||||
# GO_VERSION picks the toolchain image (default: latest stable). Test the lowest
|
|
||||||
# supported Go with: GO_VERSION=1.22 docker compose run --rm test
|
|
||||||
|
|
||||||
x-go: &go
|
x-go: &go
|
||||||
image: golang:${GO_VERSION:-1.26.4}
|
image: golang:${GO_VERSION:-1.27.2}
|
||||||
working_dir: /app
|
working_dir: /app
|
||||||
volumes:
|
volumes:
|
||||||
- .:/app
|
- .:/app
|
||||||
@@ -33,12 +17,16 @@ services:
|
|||||||
|
|
||||||
cover:
|
cover:
|
||||||
<<: *go
|
<<: *go
|
||||||
command: go test -cover ./...
|
command: sh -c "go test -coverpkg=./... -coverprofile=/tmp/cover.out ./... && go tool cover -func=/tmp/cover.out | tail -1"
|
||||||
|
|
||||||
bench:
|
bench:
|
||||||
<<: *go
|
<<: *go
|
||||||
command: go test -run XXX -bench . -benchmem ./...
|
command: go test -run XXX -bench . -benchmem ./...
|
||||||
|
|
||||||
|
generate:
|
||||||
|
<<: *go
|
||||||
|
command: go generate ./...
|
||||||
|
|
||||||
vet:
|
vet:
|
||||||
<<: *go
|
<<: *go
|
||||||
command: go vet ./...
|
command: go vet ./...
|
||||||
@@ -47,6 +35,10 @@ services:
|
|||||||
<<: *go
|
<<: *go
|
||||||
command: go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test .
|
command: go run github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 -over 14 -ignore _test .
|
||||||
|
|
||||||
|
vulncheck:
|
||||||
|
<<: *go
|
||||||
|
command: go run golang.org/x/vuln/cmd/govulncheck@v1.8.0 -scan module
|
||||||
|
|
||||||
build:
|
build:
|
||||||
<<: *go
|
<<: *go
|
||||||
command: go build ./...
|
command: go build ./...
|
||||||
@@ -59,13 +51,12 @@ services:
|
|||||||
<<: *go
|
<<: *go
|
||||||
command: go mod tidy
|
command: go mod tidy
|
||||||
|
|
||||||
# Interactive shell for ad-hoc work: docker compose run --rm dev
|
|
||||||
dev:
|
dev:
|
||||||
<<: *go
|
<<: *go
|
||||||
command: sh
|
command: sh
|
||||||
|
|
||||||
data-import:
|
data-import:
|
||||||
image: python:3.14.7-slim
|
image: python:3.14.8-slim
|
||||||
working_dir: /app
|
working_dir: /app
|
||||||
volumes:
|
volumes:
|
||||||
- .:/app
|
- .:/app
|
||||||
|
|||||||
+43
-65
@@ -1,14 +1,12 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Rebuild data/geo/SE/*.tsv from SCB (CC0), GeoNames (CC BY 4.0) and Trafikverket NVDB (CC0).
|
"""Rebuild data/geo/SE/*.tsv from SCB (CC0), GeoNames (CC BY 4.0) and Trafikverket NVDB (CC0).
|
||||||
|
|
||||||
TRAFIKVERKET_API_KEY=… data-import/geo-se.py [--key-file FILE] [--cache DIR] [--streets-per-locality N] [--out DIR]
|
TRAFIKVERKET_API_KEY=… data-import/geo-se.py [--key-file FILE] [--cache DIR] [--min-streets N] [--street-share F] [--out DIR]
|
||||||
"""
|
"""
|
||||||
import argparse
|
|
||||||
import collections
|
import collections
|
||||||
import csv
|
import csv
|
||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
import math
|
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
@@ -16,6 +14,7 @@ import urllib.request
|
|||||||
import zipfile
|
import zipfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
import geo
|
||||||
import source
|
import source
|
||||||
import tsv
|
import tsv
|
||||||
import xlsx
|
import xlsx
|
||||||
@@ -34,11 +33,15 @@ TATORTER = "https://geodata.scb.se/geoserver/stat/wfs?service=WFS&version=2.0.0&
|
|||||||
POSTAL_CODES = "https://download.geonames.org/export/zip/SE.zip"
|
POSTAL_CODES = "https://download.geonames.org/export/zip/SE.zip"
|
||||||
NVDB = "https://api.trafikinfo.trafikverket.se/v2/data.json"
|
NVDB = "https://api.trafikinfo.trafikverket.se/v2/data.json"
|
||||||
NVDB_PAGE = 50000
|
NVDB_PAGE = 50000
|
||||||
OUT = Path(__file__).resolve().parent.parent / "data" / "geo" / "SE"
|
|
||||||
CACHE = Path(__file__).resolve().parent / "cache"
|
|
||||||
TIMEZONE = "Europe/Stockholm"
|
TIMEZONE = "Europe/Stockholm"
|
||||||
|
# This script's reading of the "exceptions in big cities" in docs/research/research-geo-se.md, Postal codes → Structure.
|
||||||
ONE_POSITION = {"Stockholm", "Göteborg", "Malmö"}
|
ONE_POSITION = {"Stockholm", "Göteborg", "Malmö"}
|
||||||
|
# A tätort's floor: "minst 200 invånare", docs/research/research-geo-se.md, Lantmäteriet → Ortnamn.
|
||||||
UNMATCHED_POPULATION = 200
|
UNMATCHED_POPULATION = 200
|
||||||
|
# Not a street: a road number, a ferry route, a cycle path's or road association's label, a placeholder, or two names
|
||||||
|
# or a description in one.
|
||||||
|
NOT_A_STREET = re.compile(r"^(Europaväg|Länsväg|Riksväg) \d|^Färjeled |\b(GC|SMF|VSF)\b|^NN$|,")
|
||||||
|
PARTICLES = {"af", "de", "von"}
|
||||||
|
|
||||||
|
|
||||||
def scb_codes(cache):
|
def scb_codes(cache):
|
||||||
@@ -104,36 +107,11 @@ def nvdb_segments(cache, key):
|
|||||||
yield name, float(lat), float(lon)
|
yield name, float(lat), float(lon)
|
||||||
|
|
||||||
|
|
||||||
class Nearest:
|
|
||||||
"""Nearest point by an equirectangular distance, over a degree grid."""
|
|
||||||
|
|
||||||
def __init__(self, points, cell=0.05):
|
|
||||||
self.cell = cell
|
|
||||||
self.grid = collections.defaultdict(list)
|
|
||||||
for lat, lon, value in points:
|
|
||||||
self.grid[(int(lat // cell), int(lon // cell))].append((lat, lon, value))
|
|
||||||
|
|
||||||
def find(self, lat, lon):
|
|
||||||
ci, cj = int(lat // self.cell), int(lon // self.cell)
|
|
||||||
best, best_d = None, math.inf
|
|
||||||
ring = 0
|
|
||||||
while ring < 400:
|
|
||||||
for i in range(ci - ring, ci + ring + 1):
|
|
||||||
for j in range(cj - ring, cj + ring + 1):
|
|
||||||
if max(abs(i - ci), abs(j - cj)) != ring:
|
|
||||||
continue
|
|
||||||
for plat, plon, value in self.grid.get((i, j), ()):
|
|
||||||
d = (plat - lat) ** 2 + ((plon - lon) * math.cos(math.radians(lat))) ** 2
|
|
||||||
if d < best_d:
|
|
||||||
best, best_d = value, d
|
|
||||||
if best is not None and math.sqrt(best_d) < ring * self.cell * math.cos(math.radians(lat)):
|
|
||||||
return best
|
|
||||||
ring += 1
|
|
||||||
return best
|
|
||||||
|
|
||||||
|
|
||||||
def street_delivery(name, codes):
|
def street_delivery(name, codes):
|
||||||
"""The codes delivered to a street: the digit after the postort's own prefix says box, company or reply."""
|
"""The codes delivered to a street. A postort's own prefix is three digits where one three-digit prefix starts at
|
||||||
|
least half its codes, else two. Outside ONE_POSITION's cities, the digit after that prefix is 0 or 1 for boxes and 8
|
||||||
|
for reply mail; inside them, a second digit 0 marks boxes. The box and reply-mail digits are per
|
||||||
|
docs/research/research-geo-se.md, Postal codes → Structure."""
|
||||||
if name in ONE_POSITION:
|
if name in ONE_POSITION:
|
||||||
return [c for c in codes if c[1] != "0"]
|
return [c for c in codes if c[1] != "0"]
|
||||||
largest = collections.Counter(c[:3] for c in codes).most_common(1)[0][1]
|
largest = collections.Counter(c[:3] for c in codes).most_common(1)[0][1]
|
||||||
@@ -163,6 +141,8 @@ def population_of(name, municipality, tatorter, municipalities, population):
|
|||||||
|
|
||||||
|
|
||||||
def well_cased(name):
|
def well_cased(name):
|
||||||
|
"""Whether each word of name is capitalised: GeoNames also spells some postorter, such as STOCKHOLM and örebro, in
|
||||||
|
capitals or lower case."""
|
||||||
return all(part[:1].isupper() and (len(part) == 1 or not part.isupper()) for part in re.split(r"[ -]", name))
|
return all(part[:1].isupper() and (len(part) == 1 or not part.isupper()) for part in re.split(r"[ -]", name))
|
||||||
|
|
||||||
|
|
||||||
@@ -176,58 +156,56 @@ def localities(codes, tatorter, municipalities, population):
|
|||||||
municipality, method = municipality_of(name, rows, tatorter, municipalities)
|
municipality, method = municipality_of(name, rows, tatorter, municipalities)
|
||||||
how[method] += 1
|
how[method] += 1
|
||||||
kept = street_delivery(name, [r["code"].replace(" ", "") for r in rows])
|
kept = street_delivery(name, [r["code"].replace(" ", "") for r in rows])
|
||||||
with_point = [r for r in rows if r["lat"] is not None]
|
point = geo.centroid(rows)
|
||||||
if municipality is None or not kept or not with_point or not well_cased(name):
|
if municipality is None or not kept or point is None or not well_cased(name):
|
||||||
continue
|
continue
|
||||||
lat = sum(r["lat"] for r in with_point) / len(with_point)
|
lat, lon = point
|
||||||
lon = sum(r["lon"] for r in with_point) / len(with_point)
|
|
||||||
out[name] = {"name": name, "municipality": municipality, "population": population_of(name, municipality, tatorter, municipalities, population), "lat": f"{lat:.4f}", "lon": f"{lon:.4f}", "codes": kept}
|
out[name] = {"name": name, "municipality": municipality, "population": population_of(name, municipality, tatorter, municipalities, population), "lat": f"{lat:.4f}", "lon": f"{lon:.4f}", "codes": kept}
|
||||||
print(f"municipality by {dict(how)}; {len(by_locality) - len(out)} postorter dropped", file=sys.stderr)
|
print(f"municipality by {dict(how)}; {len(by_locality) - len(out)} postorter dropped before streets", file=sys.stderr)
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
def streets(segments, codes, localities, per_locality):
|
def street_name(name):
|
||||||
|
# A word of four capitals or fewer is an acronym or a Roman numeral: IOGT-vägen, Karl XII gatan.
|
||||||
|
opening = name[0].isupper() or name.split()[0] in PARTICLES
|
||||||
|
return opening and not NOT_A_STREET.search(name) and not any(len(w) >= 5 and w.isupper() for w in name.split())
|
||||||
|
|
||||||
|
|
||||||
|
def streets(segments, codes, localities, share, minimum):
|
||||||
"""The names with most segments per locality, each segment at its nearest code centroid."""
|
"""The names with most segments per locality, each segment at its nearest code centroid."""
|
||||||
nearest = Nearest((r["lat"], r["lon"], r["locality"]) for r in codes if r["lat"] is not None and r["locality"] in localities)
|
nearest = geo.Nearest((r["lat"], r["lon"], r["locality"]) for r in codes if r["lat"] is not None and r["locality"] in localities)
|
||||||
count = collections.Counter()
|
count = collections.Counter()
|
||||||
for name, lat, lon in segments:
|
for name, lat, lon in segments:
|
||||||
if name[0].isalpha():
|
if street_name(name):
|
||||||
count[(nearest.find(lat, lon), name)] += 1
|
count[(nearest.find(lat, lon), name)] += 1
|
||||||
of = collections.defaultdict(list)
|
return geo.top_streets(count, share, minimum, "segments")
|
||||||
for (locality, name), n in count.items():
|
|
||||||
of[locality].append((n, name))
|
|
||||||
return {locality: [{"name": name, "locality": locality, "segments": n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:per_locality]] for locality, named in of.items()}
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
a = geo.parser(__doc__, "SE", 1.0, key_file={"help": "file holding the Trafikverket API key; TRAFIKVERKET_API_KEY otherwise"}).parse_args()
|
||||||
p.add_argument("--cache", default=str(CACHE))
|
|
||||||
p.add_argument("--key-file", help="file holding the Trafikverket API key; TRAFIKVERKET_API_KEY otherwise")
|
|
||||||
p.add_argument("--out", default=str(OUT))
|
|
||||||
p.add_argument("--streets-per-locality", type=int, default=10)
|
|
||||||
a = p.parse_args()
|
|
||||||
key = Path(a.key_file).read_text().strip() if a.key_file else os.environ.get("TRAFIKVERKET_API_KEY")
|
key = Path(a.key_file).read_text().strip() if a.key_file else os.environ.get("TRAFIKVERKET_API_KEY")
|
||||||
if not key:
|
if not key:
|
||||||
sys.exit("set TRAFIKVERKET_API_KEY or pass --key-file")
|
sys.exit("set TRAFIKVERKET_API_KEY or pass --key-file")
|
||||||
cache, out = Path(a.cache), Path(a.out)
|
cache, out = geo.directories(a)
|
||||||
cache.mkdir(parents=True, exist_ok=True)
|
|
||||||
out.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
regions, municipalities = scb_codes(cache)
|
region_names, municipality_names = scb_codes(cache)
|
||||||
population = scb_population(cache)
|
population = scb_population(cache)
|
||||||
codes = geonames(cache)
|
codes = geonames(cache)
|
||||||
places = localities(codes, scb_tatorter(cache), municipalities, population)
|
places = localities(codes, scb_tatorter(cache), municipality_names, population)
|
||||||
named = streets(nvdb_segments(cache, key), codes, places, a.streets_per_locality)
|
named = streets(nvdb_segments(cache, key), codes, places, a.street_share, a.min_streets)
|
||||||
places = {name: l for name, l in places.items() if name in named}
|
regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in region_names.items()}
|
||||||
empty = sorted(m for m in municipalities if not any(l["municipality"] == m for l in places.values()))
|
municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipality_names.items()}
|
||||||
if empty:
|
kept, dropped = geo.fence(regions, municipalities, places, named, {k: l["codes"] for k, l in places.items()})
|
||||||
sys.exit(f"municipalities without a locality: {empty}")
|
geo.log_dropped_localities(dropped["locality"])
|
||||||
|
if dropped["municipality"] or dropped["region"]:
|
||||||
|
sys.exit(f"municipalities without a locality: {sorted(dropped['municipality'])}; regions without a municipality: {sorted(dropped['region'])}")
|
||||||
|
places = kept["locality"]
|
||||||
|
|
||||||
tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [{"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in sorted(regions.items())])
|
tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
|
||||||
tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [{"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in sorted(municipalities.items())])
|
tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(kept["municipality"].items())])
|
||||||
tsv.write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
|
tsv.write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
|
||||||
tsv.write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in places.values() for c in l["codes"]), key=lambda r: r["code"]))
|
tsv.write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in places.values() for c in l["codes"]), key=lambda r: r["code"]))
|
||||||
tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(named) for s in named[locality]])
|
tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(places) for s in named[locality]])
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
+76
-41
@@ -1,19 +1,21 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Rebuild data/geo/US/*.tsv from the Census Bureau's Gazetteer, population estimates, ZCTA relationships and TIGER/Line files (public domain).
|
"""Rebuild data/geo/US/*.tsv from the Census Bureau's Gazetteer, population estimates, ZCTA relationships and TIGER/Line files (public domain), and Wikidata's state capitals (CC0).
|
||||||
|
|
||||||
data-import/geo-us.py [--cache DIR] [--min-population N] [--streets-per-locality N] [--out DIR]
|
data-import/geo-us.py [--cache DIR] [--min-streets N] [--place-share F] [--street-share F] [--out DIR]
|
||||||
"""
|
"""
|
||||||
import argparse
|
|
||||||
import collections
|
import collections
|
||||||
import concurrent.futures
|
import concurrent.futures
|
||||||
import csv
|
import csv
|
||||||
import io
|
import io
|
||||||
|
import math
|
||||||
|
import random
|
||||||
import re
|
import re
|
||||||
import struct
|
import struct
|
||||||
import sys
|
import sys
|
||||||
|
import urllib.parse
|
||||||
import zipfile
|
import zipfile
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
|
import geo
|
||||||
import source
|
import source
|
||||||
import tsv
|
import tsv
|
||||||
|
|
||||||
@@ -24,14 +26,20 @@ COUNTIES = POPULATION.format("counties/totals/co-est2025-alldata.csv")
|
|||||||
PLACES = POPULATION.format("cities/totals/sub-est2025.csv")
|
PLACES = POPULATION.format("cities/totals/sub-est2025.csv")
|
||||||
ZCTA_PLACE = "https://www2.census.gov/geo/docs/maps-data/data/rel2020/zcta520/tab20_zcta520_place20_natl.txt"
|
ZCTA_PLACE = "https://www2.census.gov/geo/docs/maps-data/data/rel2020/zcta520/tab20_zcta520_place20_natl.txt"
|
||||||
TIGER = "https://www2.census.gov/geo/tiger/TIGER2025/{0}/tl_2025_{1}_{2}.zip"
|
TIGER = "https://www2.census.gov/geo/tiger/TIGER2025/{0}/tl_2025_{1}_{2}.zip"
|
||||||
OUT = Path(__file__).resolve().parent.parent / "data" / "geo" / "US"
|
|
||||||
CACHE = Path(__file__).resolve().parent / "cache"
|
|
||||||
ESTIMATE = "POPESTIMATE2025"
|
ESTIMATE = "POPESTIMATE2025"
|
||||||
CDP = "57"
|
CDP = "57"
|
||||||
|
# Each state's capital from Wikidata, with the capital's GNIS ids, which the Gazetteer carries as ANSICODE.
|
||||||
|
CAPITALS = "https://query.wikidata.org/sparql?" + urllib.parse.urlencode({"query": """
|
||||||
|
SELECT ?iso ?capitalLabel ?gnis WHERE {
|
||||||
|
?state wdt:P31 wd:Q35657; wdt:P300 ?iso; wdt:P36 ?capital.
|
||||||
|
OPTIONAL { ?capital wdt:P590 ?gnis. }
|
||||||
|
SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
|
||||||
|
}"""})
|
||||||
|
SHIPPED_POPULATION = 100000
|
||||||
HIGHWAY = re.compile(r"\b(I- |Hwy |Highway |Loop |Rte |Route |Rd )\d")
|
HIGHWAY = re.compile(r"\b(I- |Hwy |Highway |Loop |Rte |Route |Rd )\d")
|
||||||
SUFFIX = re.compile(r" (city and borough|city|town|village|borough|municipality|comunidad|zona urbana|metropolitan government|metro government|consolidated government|unified government|urban county|corporation|plantation)( \(balance\))?$")
|
SUFFIX = re.compile(r" (city and borough|city|town|village|borough|municipality|comunidad|zona urbana|metropolitan government|metro government|consolidated government|unified government|urban county|corporation|plantation)( \(balance\))?$")
|
||||||
# Places whose Census name is a merged government's; the postal city is what an address carries.
|
# Places whose Census name is not the postal city an address carries.
|
||||||
NAMES = {"1303440": "Athens", "1304204": "Augusta", "1349008": "Macon", "2148006": "Louisville", "3011397": "Butte", "4732742": "Hartsville", "4752006": "Nashville"}
|
NAMES = {"0665042": "Ventura", "1303440": "Athens", "1304204": "Augusta", "1349008": "Macon", "1608830": "Boise", "2146027": "Lexington", "2148006": "Louisville", "3011397": "Butte", "4732742": "Hartsville", "4744382": "Lynchburg", "4752006": "Nashville"}
|
||||||
TIMEZONES = {
|
TIMEZONES = {
|
||||||
"AK": "America/Anchorage", "AL": "America/Chicago", "AR": "America/Chicago", "AZ": "America/Phoenix",
|
"AK": "America/Anchorage", "AL": "America/Chicago", "AR": "America/Chicago", "AZ": "America/Phoenix",
|
||||||
"CA": "America/Los_Angeles", "CO": "America/Denver", "CT": "America/New_York", "DC": "America/New_York",
|
"CA": "America/Los_Angeles", "CO": "America/Denver", "CT": "America/New_York", "DC": "America/New_York",
|
||||||
@@ -107,8 +115,8 @@ def place_name(geoid, name):
|
|||||||
return stripped
|
return stripped
|
||||||
|
|
||||||
|
|
||||||
def localities(cache, min_population, counties):
|
def localities(cache, counties):
|
||||||
"""Each shipped place with its county, population and centroid."""
|
"""Each incorporated place with its county, population and centroid."""
|
||||||
place_population, county_part = {}, collections.defaultdict(list)
|
place_population, county_part = {}, collections.defaultdict(list)
|
||||||
for r in csv_rows(cache, PLACES, "sub-est2025.csv"):
|
for r in csv_rows(cache, PLACES, "sub-est2025.csv"):
|
||||||
if r["SUMLEV"] == "162":
|
if r["SUMLEV"] == "162":
|
||||||
@@ -118,18 +126,47 @@ def localities(cache, min_population, counties):
|
|||||||
out = {}
|
out = {}
|
||||||
for r in gazetteer(cache, "place"):
|
for r in gazetteer(cache, "place"):
|
||||||
geoid, population = r["GEOID"], place_population.get(r["GEOID"], 0)
|
geoid, population = r["GEOID"], place_population.get(r["GEOID"], 0)
|
||||||
if r["FUNCSTAT"] not in ("A", "F", "N") or r["LSAD"] == CDP or population < min_population or not county_part.get(geoid):
|
if r["FUNCSTAT"] not in ("A", "B", "F", "N") or r["LSAD"] == CDP or not county_part.get(geoid):
|
||||||
continue
|
continue
|
||||||
county = max(county_part[geoid])[1]
|
county = max(county_part[geoid])[1]
|
||||||
if county not in counties:
|
if county not in counties:
|
||||||
print(f"{geoid} {r['NAME']}: county {county} unknown, dropped", file=sys.stderr)
|
print(f"{geoid} {r['NAME']}: county {county} unknown, dropped", file=sys.stderr)
|
||||||
continue
|
continue
|
||||||
out[geoid] = {"code": geoid, "name": place_name(geoid, r["NAME"]), "municipality": county, "population": population, "lat": r["INTPTLAT"], "lon": r["INTPTLONG"]}
|
out[geoid] = {"code": geoid, "gnis": int(r["ANSICODE"]), "name": place_name(geoid, r["NAME"]), "municipality": county, "population": population, "lat": r["INTPTLAT"], "lon": r["INTPTLONG"]}
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
def postal_codes(cache, localities):
|
def capitals(cache, places, counties):
|
||||||
"""Each ZCTA whose largest part inside an incorporated place lies in a shipped place."""
|
"""The codes of the places that are a state's capital, matched by any of its GNIS ids, else by a name no other place in the state carries."""
|
||||||
|
by_gnis = {p["gnis"]: code for code, p in places.items()}
|
||||||
|
named = collections.defaultdict(list)
|
||||||
|
for code, p in places.items():
|
||||||
|
named[(counties[p["municipality"]]["region"], p["name"])].append(code)
|
||||||
|
by_name = {key: codes[0] for key, codes in named.items() if len(codes) == 1}
|
||||||
|
gnis_of = collections.defaultdict(set)
|
||||||
|
for r in csv.DictReader(io.StringIO(text(source.fetch(CAPITALS, cache, "capitals.csv", magic=b"iso,", headers={"Accept": "text/csv"})))):
|
||||||
|
gnis_of[(r["iso"].removeprefix("US-"), r["capitalLabel"])].add(int(r["gnis"] or 0))
|
||||||
|
out = set()
|
||||||
|
for (state, name), ids in sorted(gnis_of.items()):
|
||||||
|
code = next((by_gnis[i] for i in sorted(ids) if i in by_gnis), None) or by_name.get((state, name))
|
||||||
|
if code:
|
||||||
|
out.add(code)
|
||||||
|
else:
|
||||||
|
print(f"{state} {name}: capital matches no incorporated place by GNIS id or by a name unique in its state", file=sys.stderr)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def drawn(places, share):
|
||||||
|
"""The share of the places with people, each drawn with odds by its population: Efraimidis and Spirakis's weighted
|
||||||
|
sampling without replacement. Each place's random number is seeded by its code, so a place added to or dropped from
|
||||||
|
the sources moves only the places near the cut."""
|
||||||
|
peopled = [code for code, p in places.items() if p["population"] > 0]
|
||||||
|
key = {code: math.log(1 - random.Random(code).random()) / places[code]["population"] for code in peopled}
|
||||||
|
return {code: places[code] for code in sorted(peopled, key=lambda c: (key[c], c), reverse=True)[:round(len(peopled) * share)]}
|
||||||
|
|
||||||
|
|
||||||
|
def zcta_localities(cache, localities):
|
||||||
|
"""Each ZCTA mapped to the incorporated place holding most of its land inside incorporated places, where localities holds that place."""
|
||||||
parts = {}
|
parts = {}
|
||||||
for r in csv.DictReader(io.StringIO(text(source.fetch(ZCTA_PLACE, cache, "zcta-place.txt"))), delimiter="|"):
|
for r in csv.DictReader(io.StringIO(text(source.fetch(ZCTA_PLACE, cache, "zcta-place.txt"))), delimiter="|"):
|
||||||
if r["GEOID_ZCTA5_20"] and r["GEOID_PLACE_20"] and not r["NAMELSAD_PLACE_20"].endswith(" CDP"):
|
if r["GEOID_ZCTA5_20"] and r["GEOID_PLACE_20"] and not r["NAMELSAD_PLACE_20"].endswith(" CDP"):
|
||||||
@@ -138,7 +175,7 @@ def postal_codes(cache, localities):
|
|||||||
return {zcta: place for zcta, place in largest.items() if place in localities}
|
return {zcta: place for zcta, place in largest.items() if place in localities}
|
||||||
|
|
||||||
|
|
||||||
def streets(cache, counties, locality_of_zcta, per_locality):
|
def streets(cache, counties, locality_of_zcta, share, minimum):
|
||||||
"""The names with most TIGER address ranges per place, and the address ranges per ZCTA."""
|
"""The names with most TIGER address ranges per place, and the address ranges per ZCTA."""
|
||||||
with concurrent.futures.ThreadPoolExecutor(3) as pool:
|
with concurrent.futures.ThreadPoolExecutor(3) as pool:
|
||||||
list(pool.map(lambda c: (tiger_zip(cache, "addr", c), tiger_zip(cache, "featnames", c)), counties))
|
list(pool.map(lambda c: (tiger_zip(cache, "addr", c), tiger_zip(cache, "featnames", c)), counties))
|
||||||
@@ -153,23 +190,12 @@ def streets(cache, counties, locality_of_zcta, per_locality):
|
|||||||
if r["PAFLAG"] == "P" and r["FULLNAME"] and not HIGHWAY.search(r["FULLNAME"]):
|
if r["PAFLAG"] == "P" and r["FULLNAME"] and not HIGHWAY.search(r["FULLNAME"]):
|
||||||
for z in zips.get(r["TLID"], ()):
|
for z in zips.get(r["TLID"], ()):
|
||||||
count[(locality_of_zcta[z], r["FULLNAME"])] += 1
|
count[(locality_of_zcta[z], r["FULLNAME"])] += 1
|
||||||
of = collections.defaultdict(list)
|
return geo.top_streets(count, share, minimum, "addresses"), addresses
|
||||||
for (locality, name), n in count.items():
|
|
||||||
of[locality].append((n, name))
|
|
||||||
named = {locality: [{"name": name, "locality": locality, "addresses": n} for n, name in sorted(ranked, key=lambda s: (-s[0], s[1]))[:per_locality]] for locality, ranked in of.items()}
|
|
||||||
return named, addresses
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
a = geo.parser(__doc__, "US", 0.25, place_share={"type": float, "default": 0.1}).parse_args()
|
||||||
p.add_argument("--cache", default=str(CACHE))
|
cache, out = geo.directories(a)
|
||||||
p.add_argument("--min-population", type=int, default=25000)
|
|
||||||
p.add_argument("--out", default=str(OUT))
|
|
||||||
p.add_argument("--streets-per-locality", type=int, default=10)
|
|
||||||
a = p.parse_args()
|
|
||||||
cache, out = Path(a.cache), Path(a.out)
|
|
||||||
cache.mkdir(parents=True, exist_ok=True)
|
|
||||||
out.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
state_population = {r["STATE"]: r[ESTIMATE] for r in csv_rows(cache, STATES, "nst-est2025.csv") if r["SUMLEV"] == "040"}
|
state_population = {r["STATE"]: r[ESTIMATE] for r in csv_rows(cache, STATES, "nst-est2025.csv") if r["SUMLEV"] == "040"}
|
||||||
regions = {r["USPS"]: {"abbr": r["USPS"], "code": r["GEOID"], "name": r["NAME"], "population": state_population[r["GEOID"]], "timezone": TIMEZONES[r["USPS"]]} for r in gazetteer(cache, "state")}
|
regions = {r["USPS"]: {"abbr": r["USPS"], "code": r["GEOID"], "name": r["NAME"], "population": state_population[r["GEOID"]], "timezone": TIMEZONES[r["USPS"]]} for r in gazetteer(cache, "state")}
|
||||||
@@ -182,20 +208,29 @@ def main():
|
|||||||
counties[r["GEOID"]] = {"code": r["GEOID"], "name": r["NAME"], "region": r["USPS"], "population": county_population[r["GEOID"]]}
|
counties[r["GEOID"]] = {"code": r["GEOID"], "name": r["NAME"], "region": r["USPS"], "population": county_population[r["GEOID"]]}
|
||||||
print(f"counties without a population estimate, dropped: {dict(unestimated)}", file=sys.stderr)
|
print(f"counties without a population estimate, dropped: {dict(unestimated)}", file=sys.stderr)
|
||||||
|
|
||||||
places = localities(cache, a.min_population, counties)
|
places = localities(cache, counties)
|
||||||
locality_of_zcta = postal_codes(cache, places)
|
owner_of_zcta = zcta_localities(cache, places)
|
||||||
named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.streets_per_locality)
|
capital_codes = capitals(cache, places, counties)
|
||||||
for geoid in [l for l in places if l not in named]:
|
owners = {p: places[p] for p in set(owner_of_zcta.values())}
|
||||||
print(f"{geoid} {places[geoid]['name']}: no streets, dropped", file=sys.stderr)
|
for code in sorted(capital_codes - set(owners)):
|
||||||
del places[geoid]
|
print(f"{code} {places[code]['name']}: capital owns no ZCTA, dropped", file=sys.stderr)
|
||||||
kept_counties = {l["municipality"] for l in places.values()}
|
places = drawn(owners, a.place_share) | {code: p for code, p in owners.items() if code in capital_codes or p["population"] >= SHIPPED_POPULATION}
|
||||||
kept_regions = {counties[c]["region"] for c in kept_counties}
|
locality_of_zcta = {zcta: p for zcta, p in owner_of_zcta.items() if p in places}
|
||||||
|
named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.street_share, a.min_streets)
|
||||||
|
zctas = collections.defaultdict(list)
|
||||||
|
for zcta, locality in sorted(locality_of_zcta.items()):
|
||||||
|
if addresses[zcta]:
|
||||||
|
zctas[locality].append(zcta)
|
||||||
|
kept, dropped = geo.fence(regions, counties, places, named, zctas)
|
||||||
|
geo.log_dropped_localities(dropped["locality"])
|
||||||
|
print(f"counties without a place, dropped: {len(dropped['municipality'])}; states without a county, dropped: {sorted(dropped['region'])}", file=sys.stderr)
|
||||||
|
places = kept["locality"]
|
||||||
|
|
||||||
tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(regions.items()) if r["abbr"] in kept_regions])
|
tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
|
||||||
tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(counties.items()) if c["code"] in kept_counties])
|
tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(kept["municipality"].items())])
|
||||||
tsv.write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
|
tsv.write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
|
||||||
tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], [{"code": z, "locality": l, "addresses": addresses[z]} for z, l in sorted(locality_of_zcta.items()) if addresses[z] and l in places])
|
tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], sorted(({"code": z, "locality": l, "addresses": addresses[z]} for l in places for z in zctas[l]), key=lambda r: r["code"]))
|
||||||
tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(named) for s in named[locality]])
|
tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(places) for s in named[locality]])
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -0,0 +1,98 @@
|
|||||||
|
"""The steps a country's geo import may share with another's."""
|
||||||
|
import argparse
|
||||||
|
import collections
|
||||||
|
import math
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
DATA = Path(__file__).resolve().parent.parent / "data" / "geo"
|
||||||
|
CACHE = Path(__file__).resolve().parent / "cache"
|
||||||
|
|
||||||
|
|
||||||
|
def parser(doc, country, street_share, **own):
|
||||||
|
"""A parser for the options every country's import takes, plus the script's own: each keyword names one option, with
|
||||||
|
"_" standing for "-"."""
|
||||||
|
options = {"cache": {"default": str(CACHE)}, "out": {"default": str(DATA / country)}, "min_streets": {"type": int, "default": 10}, "street_share": {"type": float, "default": street_share}, **own}
|
||||||
|
p = argparse.ArgumentParser(description=doc.splitlines()[0])
|
||||||
|
for name in sorted(options):
|
||||||
|
p.add_argument("--" + name.replace("_", "-"), **options[name])
|
||||||
|
return p
|
||||||
|
|
||||||
|
|
||||||
|
def directories(args):
|
||||||
|
"""The cache and out directories of the parsed options, both created."""
|
||||||
|
cache, out = Path(args.cache), Path(args.out)
|
||||||
|
cache.mkdir(parents=True, exist_ok=True)
|
||||||
|
out.mkdir(parents=True, exist_ok=True)
|
||||||
|
return cache, out
|
||||||
|
|
||||||
|
|
||||||
|
def centroid(rows):
|
||||||
|
"""The mean lat and lon of the rows with a point, or None where no row has one."""
|
||||||
|
points = [r for r in rows if r["lat"] is not None]
|
||||||
|
if not points:
|
||||||
|
return None
|
||||||
|
return sum(r["lat"] for r in points) / len(points), sum(r["lon"] for r in points) / len(points)
|
||||||
|
|
||||||
|
|
||||||
|
class Nearest:
|
||||||
|
"""Nearest point by an equirectangular distance, over a degree grid."""
|
||||||
|
|
||||||
|
def __init__(self, points, cell=0.05):
|
||||||
|
self.cell = cell
|
||||||
|
self.grid = collections.defaultdict(list)
|
||||||
|
for lat, lon, value in points:
|
||||||
|
self.grid[(int(lat // cell), int(lon // cell))].append((lat, lon, value))
|
||||||
|
|
||||||
|
def find(self, lat, lon):
|
||||||
|
ci, cj = int(lat // self.cell), int(lon // self.cell)
|
||||||
|
best, best_d = None, math.inf
|
||||||
|
ring = 0
|
||||||
|
while ring < 400:
|
||||||
|
for i in range(ci - ring, ci + ring + 1):
|
||||||
|
for j in range(cj - ring, cj + ring + 1):
|
||||||
|
if max(abs(i - ci), abs(j - cj)) != ring:
|
||||||
|
continue
|
||||||
|
for plat, plon, value in self.grid.get((i, j), ()):
|
||||||
|
d = (plat - lat) ** 2 + ((plon - lon) * math.cos(math.radians(lat))) ** 2
|
||||||
|
if d < best_d:
|
||||||
|
best, best_d = value, d
|
||||||
|
if best is not None and math.sqrt(best_d) < ring * self.cell * math.cos(math.radians(lat)):
|
||||||
|
return best
|
||||||
|
ring += 1
|
||||||
|
return best
|
||||||
|
|
||||||
|
|
||||||
|
def top_streets(count, share, minimum, column):
|
||||||
|
"""count maps (locality, street name) to a count. Returns, per locality, rows for its most-counted names, share of
|
||||||
|
them rounded up and at least minimum, ties broken by name; each row holds its count under column."""
|
||||||
|
of = collections.defaultdict(list)
|
||||||
|
for (locality, name), n in count.items():
|
||||||
|
of[locality].append((n, name))
|
||||||
|
return {locality: [{"name": name, "locality": locality, column: n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:max(minimum, math.ceil(len(named) * share))]] for locality, named in of.items()}
|
||||||
|
|
||||||
|
|
||||||
|
def fence(regions, municipalities, localities, streets, postal_codes):
|
||||||
|
"""Meet the loader's fence that every parent row has a child row. Keeps each locality with entries in both streets
|
||||||
|
and postal_codes, then each municipality and region left with a child.
|
||||||
|
|
||||||
|
regions and municipalities map a code to a row; a municipality row names its region under "region". localities,
|
||||||
|
streets and postal_codes map a locality's key to its row, its streets and its postal codes. A locality row holds
|
||||||
|
name, municipality, population, lat and lon, plus its country's own keys.
|
||||||
|
|
||||||
|
Returns kept, then dropped: each maps "locality", "municipality" and "region" to that table's rows."""
|
||||||
|
kept = {"locality": {k: l for k, l in localities.items() if streets.get(k) and postal_codes.get(k)}}
|
||||||
|
municipality_codes = {l["municipality"] for l in kept["locality"].values()}
|
||||||
|
kept["municipality"] = {c: m for c, m in municipalities.items() if c in municipality_codes}
|
||||||
|
region_codes = {m["region"] for m in kept["municipality"].values()}
|
||||||
|
kept["region"] = {c: r for c, r in regions.items() if c in region_codes}
|
||||||
|
every = {"locality": localities, "municipality": municipalities, "region": regions}
|
||||||
|
dropped = {table: {k: row for k, row in rows.items() if k not in kept[table]} for table, rows in every.items()}
|
||||||
|
return kept, dropped
|
||||||
|
|
||||||
|
|
||||||
|
def log_dropped_localities(rows):
|
||||||
|
"""Logs each locality row as dropped, by its key, plus its name where that differs."""
|
||||||
|
for key, row in rows.items():
|
||||||
|
label = key if key == row["name"] else f"{key} {row['name']}"
|
||||||
|
print(f"{label}: no street or no postal code, dropped", file=sys.stderr)
|
||||||
@@ -23,7 +23,7 @@ CACHE = Path(__file__).resolve().parent / "cache"
|
|||||||
COLUMNS = ["idempotent", "method", "safe"]
|
COLUMNS = ["idempotent", "method", "safe"]
|
||||||
CORE = re.compile(r"\[RFC9110, Section 9\.3\.|\[RFC5789, Section 2\]")
|
CORE = re.compile(r"\[RFC9110, Section 9\.3\.|\[RFC5789, Section 2\]")
|
||||||
BOOLEAN = {"yes": "true", "no": "false"}
|
BOOLEAN = {"yes": "true", "no": "false"}
|
||||||
SELECTOR = '[]{}"|' # mirrors inSelector in table.go
|
SELECTOR = '[]{}"|' # mirrors inSelector in internal/grammar/path.go
|
||||||
EXPECTED = 9
|
EXPECTED = 9
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "protocol.tsv"
|
|||||||
CACHE = Path(__file__).resolve().parent / "cache"
|
CACHE = Path(__file__).resolve().parent / "cache"
|
||||||
COLUMNS = ["keyword", "name", "number"]
|
COLUMNS = ["keyword", "name", "number"]
|
||||||
SKIP = ("Reserved", "deprecated")
|
SKIP = ("Reserved", "deprecated")
|
||||||
SELECTOR = '[]{}"|' # mirrors inSelector in table.go
|
SELECTOR = '[]{}"|' # mirrors inSelector in internal/grammar/path.go
|
||||||
|
|
||||||
|
|
||||||
def rows(text):
|
def rows(text):
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ def fetch(source, cache, name, magic=b"", data=None, headers=None):
|
|||||||
if path.exists():
|
if path.exists():
|
||||||
return path.read_bytes()
|
return path.read_bytes()
|
||||||
for attempt in range(1, 6):
|
for attempt in range(1, 6):
|
||||||
req = urllib.request.Request(source, data=data, headers={"User-Agent": "fejkdata data-import", **(headers or {})})
|
req = urllib.request.Request(source, data=data, headers={"User-Agent": "fejkdata-data-import (https://github.com/larvit/fejkdata)", **(headers or {})})
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(req, timeout=600) as r:
|
with urllib.request.urlopen(req, timeout=600) as r:
|
||||||
body = r.read()
|
body = r.read()
|
||||||
|
|||||||
+4
-4
@@ -5,14 +5,14 @@ from pathlib import Path
|
|||||||
|
|
||||||
|
|
||||||
def write(path, columns, rows):
|
def write(path, columns, rows):
|
||||||
"""Write the rows as a TSV; the table needs two rows, and every cell must be non-empty and free of tabs, newlines and braces."""
|
"""Write the rows as a TSV of a header and at least one row, each cell stripped, non-empty and free of tabs, newlines and braces."""
|
||||||
lines = ["\t".join(columns)]
|
lines = ["\t".join(columns)]
|
||||||
for row in rows:
|
for row in rows:
|
||||||
cells = [str(row[c]) for c in columns]
|
cells = [str(row[c]).strip() for c in columns]
|
||||||
if not all(cells) or any(re.search(r"[\t\n{}]", c) for c in cells):
|
if not all(cells) or any(re.search(r"[\t\n{}]", c) for c in cells):
|
||||||
raise ValueError(f"{path}: a cell is empty or holds a tab, newline or brace: {row}")
|
raise ValueError(f"{path}: a cell is empty or holds a tab, newline or brace: {row}")
|
||||||
lines.append("\t".join(cells))
|
lines.append("\t".join(cells))
|
||||||
if len(lines) < 3:
|
if len(lines) < 2:
|
||||||
raise ValueError(f"{path}: has {len(lines) - 1} rows below its header; a table is at least two rows")
|
raise ValueError(f"{path}: has no rows below its header; a table holds at least one row")
|
||||||
Path(path).write_text("\n".join(lines) + "\n", encoding="utf-8")
|
Path(path).write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||||
print(f"{path}: {len(lines) - 1} rows", file=sys.stderr)
|
print(f"{path}: {len(lines) - 1} rows", file=sys.stderr)
|
||||||
|
|||||||
@@ -1,195 +1,175 @@
|
|||||||
package fejkdata
|
package fejkdata
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io/fs"
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata/internal/datafiles"
|
||||||
)
|
)
|
||||||
|
|
||||||
// dataSource is one tree to load: an fs.FS and the directory in it to start from.
|
// loadSources merges sources in order, the last winning a clash. A source whose manifest
|
||||||
// label prefixes file names in errors; onDisk marks path as a directory that must
|
// carries an index leaves its categories unloaded; every other source loads here, with
|
||||||
// exist.
|
// the unloaded categories it needs.
|
||||||
type dataSource struct {
|
// docs/decisions.md#a-source-whose-manifest-carries-an-index-loads-each-category-on-first-reach-any-other-loads-in-new-with-what-it-reads
|
||||||
fsys fs.FS
|
func loadSources(sources []datafiles.Source) (folder, error) {
|
||||||
label string
|
root := folder{children: map[string]node{}}
|
||||||
onDisk bool
|
|
||||||
path string
|
|
||||||
root string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s dataSource) name(p string) string {
|
|
||||||
if s.label == "" {
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
return path.Join(s.label, p)
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadData loads every source into one namespace tree and returns its root
|
|
||||||
// children. A directory becomes a folder; each *.json file in it compiles to a node
|
|
||||||
// keyed by its base name (address.json -> "address"); each subdirectory becomes a
|
|
||||||
// nested folder, so folders turn into dot-path segments. Sources merge left to right:
|
|
||||||
// matching folders merge by their children, and any other clash is won by the last
|
|
||||||
// source loaded. Once merged, linkRefs binds every reference against the
|
|
||||||
// final tree.
|
|
||||||
func loadData(sources []dataSource) (map[string]node, error) {
|
|
||||||
root := map[string]node{}
|
|
||||||
for _, src := range sources {
|
for _, src := range sources {
|
||||||
if src.onDisk {
|
m, err := src.Manifest()
|
||||||
if src.path == "" {
|
|
||||||
return nil, fmt.Errorf("a data path is empty")
|
|
||||||
}
|
|
||||||
info, err := os.Stat(src.path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("data path %s: %w", src.path, err)
|
|
||||||
}
|
|
||||||
if !info.IsDir() {
|
|
||||||
return nil, fmt.Errorf("%s is not a directory", src.path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
dir := src.root
|
|
||||||
if dir == "" {
|
|
||||||
dir = "."
|
|
||||||
}
|
|
||||||
g, err := loadDir(src, dir)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return folder{}, err
|
||||||
|
}
|
||||||
|
if err := addSource(&root, src, m); err != nil {
|
||||||
|
return folder{}, err
|
||||||
}
|
}
|
||||||
mergeChildren(root, g.children)
|
|
||||||
}
|
}
|
||||||
if len(root) == 0 {
|
return root, loadReached(&root, categorySites(&root), nil)
|
||||||
return nil, fmt.Errorf("no .json data found")
|
|
||||||
}
|
|
||||||
setTablePaths(root)
|
|
||||||
if err := linkTables(root); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := linkRefs(root); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := checkNoCycles(root); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := checkScope(treeScope(root)); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return root, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// loadDir compiles one directory into a folder. fs.ReadDir yields entries sorted
|
func addSource(root *folder, src datafiles.Source, m datafiles.Manifest) error {
|
||||||
// by name, so the tree is built deterministically. Empty subdirectories (no JSON
|
src = src.Mounted(m.Namespace)
|
||||||
// anywhere under them) are skipped rather than added as empty namespaces.
|
if m.Index != nil {
|
||||||
func loadDir(src dataSource, dir string) (*folder, error) {
|
placeIndex(root, src, m.Index)
|
||||||
entries, err := fs.ReadDir(src.fsys, dir)
|
return nil
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%s: %w", src.name(dir), err)
|
|
||||||
}
|
}
|
||||||
g := &folder{children: map[string]node{}}
|
g := &folder{children: map[string]node{}}
|
||||||
files := &categoryFiles{src: src, dir: dir, tsv: map[string]bool{}}
|
if err := src.Walk(compileInto(func(dir []string) *folder { return madeFolder(g, dir) })); err != nil {
|
||||||
for _, e := range entries {
|
|
||||||
if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() {
|
|
||||||
files.tsv[e.Name()] = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, e := range entries {
|
|
||||||
if strings.HasPrefix(e.Name(), ".") { // hidden: a checkout or an editor's file, never data
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
full := path.Join(dir, e.Name())
|
|
||||||
if e.IsDir() {
|
|
||||||
if err := loadFolder(src, g, full, e.Name()); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := loadFile(src, g, full, e.Name(), files); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for name, named := range files.tsv {
|
|
||||||
if !named && !strings.HasPrefix(name, ".") {
|
|
||||||
return nil, fmt.Errorf("%s: no category names it in its rows; a table's rows file sits beside a category file naming it", src.name(path.Join(dir, name)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return g, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// categoryFiles is what a category may name beside itself: the rows files of its
|
|
||||||
// directory, each marked once a category names it.
|
|
||||||
type categoryFiles struct {
|
|
||||||
src dataSource
|
|
||||||
dir string
|
|
||||||
tsv map[string]bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// rows reads the rows file a category names beside it.
|
|
||||||
func (c *categoryFiles) rows(name string) (string, error) {
|
|
||||||
if _, present := c.tsv[name]; !present {
|
|
||||||
return "", fmt.Errorf("rows names %s, which is not beside it in %s", name, c.src.name(c.dir))
|
|
||||||
}
|
|
||||||
c.tsv[name] = true
|
|
||||||
b, err := fs.ReadFile(c.src.fsys, path.Join(c.dir, name))
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("%s: %w", c.src.name(path.Join(c.dir, name)), err)
|
|
||||||
}
|
|
||||||
return string(b), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadFolder adds a subdirectory as a nested folder, unless nothing under it is data.
|
|
||||||
func loadFolder(src dataSource, g *folder, full, name string) error {
|
|
||||||
child, err := loadDir(src, full)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if len(child.children) == 0 {
|
mergeFolder(root, g)
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err := checkName(name); err != nil {
|
|
||||||
return fmt.Errorf("%s: folder %w", src.name(full), err)
|
|
||||||
}
|
|
||||||
g.children[name] = child
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// loadFile compiles a *.json file into a category named after it; any other file
|
func compileInto(place func(dir []string) *folder) func(datafiles.Category) error {
|
||||||
// is skipped.
|
return func(c datafiles.Category) error {
|
||||||
func loadFile(src dataSource, g *folder, full, file string, files *categoryFiles) error {
|
n, err := compileCategory(c)
|
||||||
if !strings.HasSuffix(file, ".json") {
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
place(c.Folders).put(c.Name, n)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
name := strings.TrimSuffix(file, ".json")
|
|
||||||
if err := checkName(name); err != nil {
|
|
||||||
return fmt.Errorf("%s: category %w", src.name(full), err)
|
|
||||||
}
|
|
||||||
b, err := fs.ReadFile(src.fsys, full)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("%s: %w", src.name(full), err)
|
|
||||||
}
|
|
||||||
var raw any
|
|
||||||
if err := json.Unmarshal(b, &raw); err != nil {
|
|
||||||
return fmt.Errorf("%s: %w", src.name(full), err)
|
|
||||||
}
|
|
||||||
n, err := compileCategory(raw, name, files)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("%s: %w", src.name(full), err)
|
|
||||||
}
|
|
||||||
g.children[name] = n
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// mergeChildren overlays src onto dst. Two folders under the same key merge
|
// madeFolder is the folder that dir names under root, made where it is missing or a category.
|
||||||
// recursively (so locales/categories from several paths combine); every other
|
func madeFolder(root *folder, dir []string) *folder {
|
||||||
// key is replaced, making the last-loaded directory win on a conflict.
|
g := root
|
||||||
func mergeChildren(dst, src map[string]node) {
|
for _, seg := range dir {
|
||||||
for k, v := range src {
|
sub, isFolder := g.children[seg].(*folder)
|
||||||
if dg, ok := dst[k].(*folder); ok {
|
if !isFolder {
|
||||||
|
sub = &folder{children: map[string]node{}}
|
||||||
|
g.put(seg, sub)
|
||||||
|
}
|
||||||
|
g = sub
|
||||||
|
}
|
||||||
|
return g
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeFolder(dst, src *folder) {
|
||||||
|
for k, v := range src.children {
|
||||||
|
if dg, ok := dst.children[k].(*folder); ok {
|
||||||
if sg, ok := v.(*folder); ok {
|
if sg, ok := v.(*folder); ok {
|
||||||
mergeChildren(dg.children, sg.children)
|
mergeFolder(dg, sg)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
dst[k] = v
|
dst.put(k, v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// nodeSet is the set of nodes one validation pass covers: one load's categories, or a
|
||||||
|
// single inline node.
|
||||||
|
type nodeSet func(fn func(label string, n node) error) error
|
||||||
|
|
||||||
|
// categorySite is a loaded category, where it sits and its own dot path.
|
||||||
|
type categorySite struct {
|
||||||
|
categoryAt
|
||||||
|
n node
|
||||||
|
path string
|
||||||
|
}
|
||||||
|
|
||||||
|
// categorySites lists every category under root, folders and names in sorted order.
|
||||||
|
func categorySites(root *folder) []categorySite {
|
||||||
|
var out []categorySite
|
||||||
|
var walk func(dir []string, g *folder)
|
||||||
|
walk = func(dir []string, g *folder) {
|
||||||
|
for _, name := range sortedNames(g.children) {
|
||||||
|
if sub, isFolder := g.children[name].(*folder); isFolder {
|
||||||
|
walk(append(dir[:len(dir):len(dir)], name), sub)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, siteIn(dir, g, name))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
walk(nil, root)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func siteIn(dir []string, in *folder, name string) categorySite {
|
||||||
|
return categorySite{categoryAt: categoryAt{dir: dir, in: in, name: name}, n: in.children[name], path: categoryPath(dir, name)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func categoryPath(dir []string, name string) string { return join(strings.Join(dir, "."), name) }
|
||||||
|
|
||||||
|
func siteNodes(sites []categorySite) nodeSet {
|
||||||
|
return func(fn func(label string, n node) error) error {
|
||||||
|
for _, s := range sites {
|
||||||
|
if err := eachNode(s.n, s.path, fn); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func inlineNodes(n node, label string) nodeSet {
|
||||||
|
return func(fn func(label string, m node) error) error { return eachNode(n, label, fn) }
|
||||||
|
}
|
||||||
|
|
||||||
|
// pipeline is the one load pipeline, run over one nodeSet; its fields are where
|
||||||
|
// entry points differ.
|
||||||
|
type pipeline struct {
|
||||||
|
nodes nodeSet
|
||||||
|
resolve func() error
|
||||||
|
// Set by a caller that proves the columns against their Go types itself.
|
||||||
|
typedByGo bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// categoryPipeline is the pipeline over the categories of one load, their references
|
||||||
|
// resolving against root.
|
||||||
|
func categoryPipeline(sites []categorySite, root map[string]node) pipeline {
|
||||||
|
return pipeline{
|
||||||
|
nodes: siteNodes(sites),
|
||||||
|
resolve: func() error {
|
||||||
|
if err := linkTables(sites); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return resolveCategoryTemplates(sites, root)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p pipeline) run() error {
|
||||||
|
if err := p.resolve(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := checkNoCycles(p.nodes); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
settleRecords(p.nodes)
|
||||||
|
if !p.typedByGo {
|
||||||
|
if err := checkColumns(p.nodes); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return checkNodeFences(p.nodes)
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkNodeFences runs the per-node fences over s, each over every node before the next,
|
||||||
|
// so which of several broken nodes is reported does not depend on the walk. Its walks
|
||||||
|
// recurse unguarded, so pipeline.run refuses cycles first.
|
||||||
|
func checkNodeFences(s nodeSet) error {
|
||||||
|
mem := renderCounts{}
|
||||||
|
if err := s(func(label string, n node) error { return repeatCheck(label, n, mem) }); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return s((&valueProof{}).checkDatatype)
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
{
|
||||||
|
"index": {
|
||||||
|
"en_US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
|
||||||
|
"en_US.color": {"paths":[""]},
|
||||||
|
"en_US.company": {"paths":["","base","suffix"]},
|
||||||
|
"en_US.date": {"paths":[""]},
|
||||||
|
"en_US.email": {"paths":["","domain","local","local.n"]},
|
||||||
|
"en_US.first-name": {"parent":"sex","paths":["","count","name","sex"]},
|
||||||
|
"en_US.ip": {"paths":[""]},
|
||||||
|
"en_US.itin": {"paths":["","area","group","serial"]},
|
||||||
|
"en_US.last-name": {"paths":["","count","name"]},
|
||||||
|
"en_US.person": {"paths":["","first","last","prefix","sex"]},
|
||||||
|
"en_US.phone": {"paths":["","area","exchange","line"]},
|
||||||
|
"en_US.phone-exchange": {"paths":[""]},
|
||||||
|
"en_US.price": {"paths":["","amt","cents"]},
|
||||||
|
"en_US.sentence": {"paths":["","adj","noun","prep","verb"]},
|
||||||
|
"en_US.sex": {"paths":["","code","first-name","first-name.count","first-name.name","first-name.sex","name","title","title.name","title.sex","title.share"]},
|
||||||
|
"en_US.ssn": {"paths":["","area","group","serial"]},
|
||||||
|
"en_US.time": {"paths":[""]},
|
||||||
|
"en_US.title": {"parent":"sex","paths":["","name","sex","share"]},
|
||||||
|
"en_US.url": {"paths":["","host","path"]},
|
||||||
|
"en_US.username": {"paths":["","n"]},
|
||||||
|
"en_US.version": {"paths":["","n","pre","suffix"]},
|
||||||
|
"en_US.word": {"paths":[""]}
|
||||||
|
},
|
||||||
|
"namespace": "en_US"
|
||||||
|
}
|
||||||
@@ -1,8 +1,8 @@
|
|||||||
{
|
{
|
||||||
"format": "{street-number} {street}\n{locality}, {region} {postal-code}",
|
"format": "{/geo.US.address as a}{a}",
|
||||||
"street": "{/geo.US.address.street}",
|
"street": "{a.street}",
|
||||||
"street-number": "{/geo.US.address.street-number}",
|
"street-number": "{a.street-number}",
|
||||||
"locality": "{/geo.US.address.locality}",
|
"locality": "{a.locality}",
|
||||||
"region": "{/geo.US.address.region}",
|
"region": "{a.region}",
|
||||||
"postal-code": "{/geo.US.address.postal-code}"
|
"postal-code": "{a.postal-code}"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
// Package enus is fejkdata's en_US locale module, whose categories are en_US.*; its address reads
|
||||||
|
// geo.US, which package geous holds.
|
||||||
|
package enus
|
||||||
|
|
||||||
|
import (
|
||||||
|
"embed"
|
||||||
|
"io/fs"
|
||||||
|
|
||||||
|
geous "github.com/larvit/fejkdata/data/geo/US"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FS holds the locale alone, its categories and its manifest; pass Modules unless your own
|
||||||
|
// data stands in for geo.US.
|
||||||
|
//
|
||||||
|
//go:embed *.json *.tsv .fejkdata.json
|
||||||
|
var FS embed.FS
|
||||||
|
|
||||||
|
// Modules returns FS and every module it reads, to pass to fejkdata.WithDataFS.
|
||||||
|
func Modules() []fs.FS { return append([]fs.FS{FS}, geous.Modules()...) }
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package enus_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata"
|
||||||
|
enus "github.com/larvit/fejkdata/data/en_US"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFSLoadsByItsManifest(t *testing.T) {
|
||||||
|
if _, err := fs.Stat(enus.FS, ".fejkdata.json"); err != nil {
|
||||||
|
t.Fatalf("FS holds no manifest: %v", err)
|
||||||
|
}
|
||||||
|
f, err := fejkdata.New(fejkdata.WithDataFS(enus.Modules()...), fejkdata.WithSeed(1))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
|
||||||
|
}
|
||||||
|
if len(f.List()) == 0 {
|
||||||
|
t.Error("List() is empty, want the module's categories")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"format": "{prefix}{first} {last}",
|
"format": "{prefix}{first} {last}",
|
||||||
"first": "{.first-name.name}",
|
"first": "{.first-name as n}{n.name}",
|
||||||
"last": "{.last-name.name}",
|
"last": "{.last-name.name}",
|
||||||
"prefix": [null, { "format": "{.title.name} ", "weight": 0.1 }],
|
"prefix": [null, { "format": "{n..sex.title.name} ", "weight": 0.1 }],
|
||||||
"sex": "{.sex.name}"
|
"sex": "{n..sex.name}"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
[
|
||||||
|
{ "format": "{int(2,9)}0{digits(1)}", "weight": 80 },
|
||||||
|
{ "format": "{int(2,9)}1{int(2,9)}", "weight": 64 },
|
||||||
|
{ "format": "{int(2,9)}10", "weight": 8 },
|
||||||
|
{ "format": "{int(2,4)}{int(2,9)}{digits(1)}", "weight": 240 },
|
||||||
|
{ "format": "{int(6,8)}{int(2,9)}{digits(1)}", "weight": 240 },
|
||||||
|
{ "format": "5{int(2,4)}{digits(1)}", "weight": 30 },
|
||||||
|
{ "format": "5{int(6,9)}{digits(1)}", "weight": 40 },
|
||||||
|
{ "format": "55{int(0,4)}", "weight": 5 },
|
||||||
|
{ "format": "55{int(6,9)}", "weight": 4 },
|
||||||
|
{ "format": "9{int(2,4)}{digits(1)}", "weight": 30 },
|
||||||
|
{ "format": "9{int(8,9)}{digits(1)}", "weight": 20 },
|
||||||
|
{ "format": "95{int(1,7)}", "weight": 7 },
|
||||||
|
{ "format": "96{digits(1)}", "weight": 10 },
|
||||||
|
{ "format": "97{int(0,5)}", "weight": 6 },
|
||||||
|
{ "format": "97{int(7,9)}", "weight": 3 }
|
||||||
|
]
|
||||||
@@ -1,15 +1,15 @@
|
|||||||
[
|
[
|
||||||
{
|
{
|
||||||
"format": "({area}) {exch}-{line}",
|
"format": "({area}) {exchange}-{line}",
|
||||||
"weight": 2,
|
"weight": 2,
|
||||||
"area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
|
"area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
|
||||||
"exch": "{int(100,999)}",
|
"exchange": "{.phone-exchange}",
|
||||||
"line": "{digits(4)}"
|
"line": "{digits(4)}"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"format": "{area}-{exch}-{line}",
|
"format": "{area}-{exchange}-{line}",
|
||||||
"area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
|
"area": ["202", "212", "213", "305", "312", "404", "415", "469", "503", "512", "602", "617", "646", "702", "718", "720", "737", "773", "786", "808", "917"],
|
||||||
"exch": "{int(100,999)}",
|
"exchange": "{.phone-exchange}",
|
||||||
"line": "{digits(4)}"
|
"line": "{digits(4)}"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
{
|
||||||
|
"index": {
|
||||||
|
"geo.SE.address": {"paths":["","locality","postal-code","street","street-number"]},
|
||||||
|
"geo.SE.locality": {"parent":"municipality","paths":["","lat","lon","municipality","name","population","postal-code","postal-code.code","postal-code.locality","street","street.locality","street.name","street.segments"]},
|
||||||
|
"geo.SE.municipality": {"parent":"region","paths":["","code","locality","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.locality","locality.street.name","locality.street.segments","name","population","region"]},
|
||||||
|
"geo.SE.postal-code": {"parent":"locality","paths":["","code","locality"]},
|
||||||
|
"geo.SE.region": {"paths":["","code","municipality","municipality.code","municipality.locality","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.locality","municipality.locality.street.name","municipality.locality.street.segments","municipality.name","municipality.population","municipality.region","name","population","timezone"]},
|
||||||
|
"geo.SE.street": {"parent":"locality","paths":["","locality","name","segments"]}
|
||||||
|
},
|
||||||
|
"namespace": "geo.SE"
|
||||||
|
}
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"format": "{street} {street-number}\n{postal-code} {locality}",
|
"format": "{street} {street-number}\n{postal-code} {locality}",
|
||||||
"street": "{.street.name}",
|
"street": "{.locality as l}{l.street.name}",
|
||||||
"street-number": [
|
"street-number": [
|
||||||
"{int(1,9)}",
|
"{int(1,9)}",
|
||||||
"{int(10,99)}",
|
"{int(10,99)}",
|
||||||
@@ -9,6 +9,6 @@
|
|||||||
{ "format": "{int(10,99)}{upper(1)}", "weight": 0.1 },
|
{ "format": "{int(10,99)}{upper(1)}", "weight": 0.1 },
|
||||||
{ "format": "{int(100,999)}{upper(1)}", "weight": 0.05 }
|
{ "format": "{int(100,999)}{upper(1)}", "weight": 0.05 }
|
||||||
],
|
],
|
||||||
"postal-code": "{.postal-code.code}",
|
"postal-code": "{l.postal-code.code}",
|
||||||
"locality": "{.locality.name}"
|
"locality": "{l.name}"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,15 @@
|
|||||||
|
// Package geose is fejkdata's Swedish places module, whose categories are geo.SE.*.
|
||||||
|
package geose
|
||||||
|
|
||||||
|
import (
|
||||||
|
"embed"
|
||||||
|
"io/fs"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FS holds the module's categories and its manifest.
|
||||||
|
//
|
||||||
|
//go:embed *.json *.tsv .fejkdata.json
|
||||||
|
var FS embed.FS
|
||||||
|
|
||||||
|
// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
|
||||||
|
func Modules() []fs.FS { return []fs.FS{FS} }
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package geose_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata"
|
||||||
|
geose "github.com/larvit/fejkdata/data/geo/SE"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFSLoadsByItsManifest(t *testing.T) {
|
||||||
|
if _, err := fs.Stat(geose.FS, ".fejkdata.json"); err != nil {
|
||||||
|
t.Fatalf("FS holds no manifest: %v", err)
|
||||||
|
}
|
||||||
|
f, err := fejkdata.New(fejkdata.WithDataFS(geose.Modules()...), fejkdata.WithSeed(1))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
|
||||||
|
}
|
||||||
|
if len(f.List()) == 0 {
|
||||||
|
t.Error("List() is empty, want the module's categories")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -866,7 +866,6 @@ Nynäshamn 0192 16447 58.9034 17.9479
|
|||||||
Nyvång 1277 200 56.1333 12.9000
|
Nyvång 1277 200 56.1333 12.9000
|
||||||
Nyåker 2403 200 63.7500 19.3167
|
Nyåker 2403 200 63.7500 19.3167
|
||||||
Nälden 2309 986 63.3500 14.2500
|
Nälden 2309 986 63.3500 14.2500
|
||||||
Nämdö 0120 200 59.1833 18.6833
|
|
||||||
Nässjö 0682 18639 57.6531 14.6968
|
Nässjö 0682 18639 57.6531 14.6968
|
||||||
Näsum 1272 1274 56.1667 14.4833
|
Näsum 1272 1274 56.1667 14.4833
|
||||||
Näsviken 2184 994 61.7667 16.8667
|
Näsviken 2184 994 61.7667 16.8667
|
||||||
|
|||||||
|
@@ -949,7 +949,6 @@ code locality
|
|||||||
130 25 Ingmarsö
|
130 25 Ingmarsö
|
||||||
130 33 Gällnöby
|
130 33 Gällnöby
|
||||||
130 34 Skälvik
|
130 34 Skälvik
|
||||||
130 36 Nämdö
|
|
||||||
130 37 Stavsnäs
|
130 37 Stavsnäs
|
||||||
130 38 Runmarö
|
130 38 Runmarö
|
||||||
130 39 Sandhamn
|
130 39 Sandhamn
|
||||||
|
|||||||
|
+159557
-5
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,11 @@
|
|||||||
|
{
|
||||||
|
"index": {
|
||||||
|
"geo.US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
|
||||||
|
"geo.US.locality": {"parent":"municipality","paths":["","code","lat","lon","municipality","name","population","postal-code","postal-code.addresses","postal-code.code","postal-code.locality","street","street.addresses","street.locality","street.name"]},
|
||||||
|
"geo.US.municipality": {"parent":"region","paths":["","code","locality","locality.code","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.addresses","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.addresses","locality.street.locality","locality.street.name","name","population","region"]},
|
||||||
|
"geo.US.postal-code": {"parent":"locality","paths":["","addresses","code","locality"]},
|
||||||
|
"geo.US.region": {"paths":["","abbr","code","municipality","municipality.code","municipality.locality","municipality.locality.code","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.addresses","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.addresses","municipality.locality.street.locality","municipality.locality.street.name","municipality.name","municipality.population","municipality.region","name","population","timezone"]},
|
||||||
|
"geo.US.street": {"parent":"locality","paths":["","addresses","locality","name"]}
|
||||||
|
},
|
||||||
|
"namespace": "geo.US"
|
||||||
|
}
|
||||||
@@ -1,13 +1,13 @@
|
|||||||
{
|
{
|
||||||
"format": "{street-number} {street}\n{locality}, {region} {postal-code}",
|
"format": "{street-number} {street}\n{locality}, {region} {postal-code}",
|
||||||
"street": "{.street.name}",
|
"street": "{.locality as l}{l.street.name}",
|
||||||
"street-number": [
|
"street-number": [
|
||||||
"{int(10,99)}",
|
"{int(10,99)}",
|
||||||
{ "format": "{int(100,999)}", "weight": 2 },
|
{ "format": "{int(100,999)}", "weight": 2 },
|
||||||
{ "format": "{int(1000,9999)}", "weight": 0.5 },
|
{ "format": "{int(1000,9999)}", "weight": 0.5 },
|
||||||
{ "format": "{int(10,99)}{int(100,999)}", "weight": 0.3 }
|
{ "format": "{int(10,99)}{int(100,999)}", "weight": 0.3 }
|
||||||
],
|
],
|
||||||
"locality": "{.locality.name}",
|
"locality": "{l.name}",
|
||||||
"region": "{.region.abbr}",
|
"region": "{l..municipality..region.abbr}",
|
||||||
"postal-code": "{.postal-code.code}"
|
"postal-code": "{l.postal-code.code}"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,15 @@
|
|||||||
|
// Package geous is fejkdata's US places module, whose categories are geo.US.*.
|
||||||
|
package geous
|
||||||
|
|
||||||
|
import (
|
||||||
|
"embed"
|
||||||
|
"io/fs"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FS holds the module's categories and its manifest.
|
||||||
|
//
|
||||||
|
//go:embed *.json *.tsv .fejkdata.json
|
||||||
|
var FS embed.FS
|
||||||
|
|
||||||
|
// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
|
||||||
|
func Modules() []fs.FS { return []fs.FS{FS} }
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package geous_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata"
|
||||||
|
geous "github.com/larvit/fejkdata/data/geo/US"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFSLoadsByItsManifest(t *testing.T) {
|
||||||
|
if _, err := fs.Stat(geous.FS, ".fejkdata.json"); err != nil {
|
||||||
|
t.Fatalf("FS holds no manifest: %v", err)
|
||||||
|
}
|
||||||
|
f, err := fejkdata.New(fejkdata.WithDataFS(geous.Modules()...), fejkdata.WithSeed(1))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
|
||||||
|
}
|
||||||
|
if len(f.List()) == 0 {
|
||||||
|
t.Error("List() is empty, want the module's categories")
|
||||||
|
}
|
||||||
|
}
|
||||||
+717
-585
File diff suppressed because it is too large
Load Diff
+353
-132
File diff suppressed because it is too large
Load Diff
+1113
-1389
File diff suppressed because it is too large
Load Diff
+465333
-5902
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"index": {
|
||||||
|
"misc.car": {"paths":["","make","model"]},
|
||||||
|
"misc.coordinate": {"paths":["","lat","lon"]},
|
||||||
|
"misc.creditcard": {"paths":["","d"]},
|
||||||
|
"misc.currency": {"paths":["","code","decimals","name","numeric","symbol"]},
|
||||||
|
"misc.datetime": {"paths":[""]},
|
||||||
|
"misc.emoji": {"paths":[""]},
|
||||||
|
"misc.httpmethod": {"paths":["","idempotent","method","safe"]},
|
||||||
|
"misc.httpstatus": {"paths":["","code","reason"]},
|
||||||
|
"misc.language": {"paths":["","code","code3","name"]},
|
||||||
|
"misc.loglevel": {"paths":["","code","keyword","severity"]},
|
||||||
|
"misc.mac": {"paths":[""]},
|
||||||
|
"misc.mimetype": {"paths":["","ext","type"]},
|
||||||
|
"misc.objectid": {"paths":[""]},
|
||||||
|
"misc.port": {"paths":["","number","service"]},
|
||||||
|
"misc.protocol": {"paths":["","keyword","name","number"]},
|
||||||
|
"misc.territory": {"paths":["","alpha2","alpha3","calling-code","capital","country","currency","flag","languages","name","numeric","timezone","timezone.offset","timezone.territory","timezone.weight","timezone.zone","tld"]},
|
||||||
|
"misc.timezone": {"parent":"territory","paths":["","offset","territory","weight","zone"]},
|
||||||
|
"misc.tld": {"paths":["","tld","type","unicode"]},
|
||||||
|
"misc.useragent": {"paths":["","browser","device","os","ua"]},
|
||||||
|
"misc.uuid": {"paths":["","variant"]}
|
||||||
|
},
|
||||||
|
"namespace": "misc"
|
||||||
|
}
|
||||||
@@ -71,7 +71,7 @@ JPY 0 Yen 392 ¥
|
|||||||
KES 2 Kenyan Shilling 404 KES
|
KES 2 Kenyan Shilling 404 KES
|
||||||
KGS 2 Som 417 ⃀
|
KGS 2 Som 417 ⃀
|
||||||
KHR 2 Riel 116 ៛
|
KHR 2 Riel 116 ៛
|
||||||
KMF 0 Comorian Franc 174 CF
|
KMF 0 Comorian Franc 174 CF
|
||||||
KPW 2 North Korean Won 408 ₩
|
KPW 2 North Korean Won 408 ₩
|
||||||
KRW 0 Won 410 ₩
|
KRW 0 Won 410 ₩
|
||||||
KWD 3 Kuwaiti Dinar 414 KWD
|
KWD 3 Kuwaiti Dinar 414 KWD
|
||||||
|
|||||||
|
@@ -0,0 +1,15 @@
|
|||||||
|
// Package misc is fejkdata's locale-neutral module, whose categories are misc.*.
|
||||||
|
package misc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"embed"
|
||||||
|
"io/fs"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FS holds the module's categories and its manifest.
|
||||||
|
//
|
||||||
|
//go:embed *.json *.tsv .fejkdata.json
|
||||||
|
var FS embed.FS
|
||||||
|
|
||||||
|
// Modules returns FS, which reads no other module, to pass to fejkdata.WithDataFS.
|
||||||
|
func Modules() []fs.FS { return []fs.FS{FS} }
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package misc_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata"
|
||||||
|
"github.com/larvit/fejkdata/data/misc"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFSLoadsByItsManifest(t *testing.T) {
|
||||||
|
if _, err := fs.Stat(misc.FS, ".fejkdata.json"); err != nil {
|
||||||
|
t.Fatalf("FS holds no manifest: %v", err)
|
||||||
|
}
|
||||||
|
f, err := fejkdata.New(fejkdata.WithDataFS(misc.Modules()...), fejkdata.WithSeed(1))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
|
||||||
|
}
|
||||||
|
if len(f.List()) == 0 {
|
||||||
|
t.Error("List() is empty, want the module's categories")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -48,7 +48,7 @@ CO COL 57 Bogota CO COP 🇨🇴 es-CO Colombia 170 .co
|
|||||||
CR CRI 506 San Jose CR CRC 🇨🇷 es-CR,en Costa Rica 188 .cr
|
CR CRI 506 San Jose CR CRC 🇨🇷 es-CR,en Costa Rica 188 .cr
|
||||||
CU CUB 53 Havana CU CUP 🇨🇺 es-CU,pap Cuba 192 .cu
|
CU CUB 53 Havana CU CUP 🇨🇺 es-CU,pap Cuba 192 .cu
|
||||||
CV CPV 238 Praia CV CVE 🇨🇻 pt-CV Cape Verde 132 .cv
|
CV CPV 238 Praia CV CVE 🇨🇻 pt-CV Cape Verde 132 .cv
|
||||||
CW CUW 599 Willemstad NL XCG 🇨🇼 nl,pap Curaçao 531 .cw
|
CW CUW 599 Willemstad NL XCG 🇨🇼 nl,pap Curaçao 531 .cw
|
||||||
CX CXR 61 Flying Fish Cove AU AUD 🇨🇽 en,zh,ms-CX Christmas Island 162 .cx
|
CX CXR 61 Flying Fish Cove AU AUD 🇨🇽 en,zh,ms-CX Christmas Island 162 .cx
|
||||||
CY CYP 357 Nicosia CY EUR 🇨🇾 el-CY,tr-CY,en Cyprus 196 .cy
|
CY CYP 357 Nicosia CY EUR 🇨🇾 el-CY,tr-CY,en Cyprus 196 .cy
|
||||||
CZ CZE 420 Prague CZ CZK 🇨🇿 cs,sk Czechia 203 .cz
|
CZ CZE 420 Prague CZ CZK 🇨🇿 cs,sk Czechia 203 .cz
|
||||||
|
|||||||
|
@@ -0,0 +1,19 @@
|
|||||||
|
// Package data lists every module fejkdata ships.
|
||||||
|
package data
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
|
||||||
|
enus "github.com/larvit/fejkdata/data/en_US"
|
||||||
|
geose "github.com/larvit/fejkdata/data/geo/SE"
|
||||||
|
geous "github.com/larvit/fejkdata/data/geo/US"
|
||||||
|
"github.com/larvit/fejkdata/data/misc"
|
||||||
|
svse "github.com/larvit/fejkdata/data/sv_SE"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:generate env REGENERATE=1 go test -run ^TestShippedManifestsAreCurrent$ ..
|
||||||
|
|
||||||
|
// Modules returns every module fejkdata ships, to pass to fejkdata.WithDataFS.
|
||||||
|
func Modules() []fs.FS {
|
||||||
|
return []fs.FS{enus.FS, geose.FS, geous.FS, misc.FS, svse.FS}
|
||||||
|
}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
{
|
||||||
|
"index": {
|
||||||
|
"sv_SE.address": {"paths":["","locality","postal-code","street","street-number"]},
|
||||||
|
"sv_SE.birth-number": {"parent":"sex","paths":["","number","sex"]},
|
||||||
|
"sv_SE.color": {"paths":[""]},
|
||||||
|
"sv_SE.company": {"paths":["","base","suffix"]},
|
||||||
|
"sv_SE.date": {"paths":[""]},
|
||||||
|
"sv_SE.email": {"paths":["","domain","local","local.n"]},
|
||||||
|
"sv_SE.first-name": {"parent":"sex","paths":["","count","name","sex"]},
|
||||||
|
"sv_SE.ip": {"paths":[""]},
|
||||||
|
"sv_SE.last-name": {"paths":["","count","name"]},
|
||||||
|
"sv_SE.person": {"paths":["","first","last","prefix","sex"]},
|
||||||
|
"sv_SE.personnummer": {"paths":[""]},
|
||||||
|
"sv_SE.phone": {"paths":["","a","b","c","prefix"]},
|
||||||
|
"sv_SE.price": {"paths":["","amt","ore"]},
|
||||||
|
"sv_SE.samordningsnummer": {"paths":[""]},
|
||||||
|
"sv_SE.sentence": {"paths":["","adj","noun","prep","verb"]},
|
||||||
|
"sv_SE.sex": {"paths":["","birth-number","birth-number.number","birth-number.sex","code","first-name","first-name.count","first-name.name","first-name.sex","name"]},
|
||||||
|
"sv_SE.time": {"paths":[""]},
|
||||||
|
"sv_SE.title": {"paths":["","name","share"]},
|
||||||
|
"sv_SE.url": {"paths":["","host","path"]},
|
||||||
|
"sv_SE.username": {"paths":["","n"]},
|
||||||
|
"sv_SE.version": {"paths":["","n","pre","suffix"]},
|
||||||
|
"sv_SE.word": {"paths":[""]}
|
||||||
|
},
|
||||||
|
"namespace": "sv_SE"
|
||||||
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"format": "{street} {street-number}\n{postal-code} {locality}",
|
"format": "{/geo.SE.address as a}{a}",
|
||||||
"street": "{/geo.SE.address.street}",
|
"street": "{a.street}",
|
||||||
"street-number": "{/geo.SE.address.street-number}",
|
"street-number": "{a.street-number}",
|
||||||
"postal-code": "{/geo.SE.address.postal-code}",
|
"postal-code": "{a.postal-code}",
|
||||||
"locality": "{/geo.SE.address.locality}"
|
"locality": "{a.locality}"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
// Package svse is fejkdata's sv_SE locale module, whose categories are sv_SE.*; its address reads
|
||||||
|
// geo.SE, which package geose holds.
|
||||||
|
package svse
|
||||||
|
|
||||||
|
import (
|
||||||
|
"embed"
|
||||||
|
"io/fs"
|
||||||
|
|
||||||
|
geose "github.com/larvit/fejkdata/data/geo/SE"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FS holds the locale alone, its categories and its manifest; pass Modules unless your own
|
||||||
|
// data stands in for geo.SE.
|
||||||
|
//
|
||||||
|
//go:embed *.json *.tsv .fejkdata.json
|
||||||
|
var FS embed.FS
|
||||||
|
|
||||||
|
// Modules returns FS and every module it reads, to pass to fejkdata.WithDataFS.
|
||||||
|
func Modules() []fs.FS { return append([]fs.FS{FS}, geose.Modules()...) }
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package svse_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata"
|
||||||
|
svse "github.com/larvit/fejkdata/data/sv_SE"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFSLoadsByItsManifest(t *testing.T) {
|
||||||
|
if _, err := fs.Stat(svse.FS, ".fejkdata.json"); err != nil {
|
||||||
|
t.Fatalf("FS holds no manifest: %v", err)
|
||||||
|
}
|
||||||
|
f, err := fejkdata.New(fejkdata.WithDataFS(svse.Modules()...), fejkdata.WithSeed(1))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New(WithDataFS(Modules()...)) = %v", err)
|
||||||
|
}
|
||||||
|
if len(f.List()) == 0 {
|
||||||
|
t.Error("List() is empty, want the module's categories")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"format": "{prefix}{first} {last}",
|
"format": "{prefix}{first} {last}",
|
||||||
"first": "{.first-name.name}",
|
"first": "{.first-name as n}{n.name}",
|
||||||
"last": "{.last-name.name}",
|
"last": "{.last-name.name}",
|
||||||
"prefix": [null, { "format": "{.title.name} ", "weight": 0.05 }],
|
"prefix": [null, { "format": "{.title.name} ", "weight": 0.05 }],
|
||||||
"sex": "{.sex.name}"
|
"sex": "{n..sex.name}"
|
||||||
}
|
}
|
||||||
|
|||||||
+243
-38
@@ -1,7 +1,10 @@
|
|||||||
package fejkdata
|
package fejkdata
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"slices"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -48,7 +51,7 @@ func TestShippedDataCategories(t *testing.T) {
|
|||||||
regexp.MustCompile(`^\d{1,3}( \d{3})?(,\d{2})? kr$`)},
|
regexp.MustCompile(`^\d{1,3}( \d{3})?(,\d{2})? kr$`)},
|
||||||
}
|
}
|
||||||
|
|
||||||
f := newGenerator(t, "data", WithSeed(1))
|
f := shipped(t, WithSeed(1))
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
for i := 0; i < 200; i++ {
|
for i := 0; i < 200; i++ {
|
||||||
if v := fake(t, f, "en_US."+c.path); !c.en.MatchString(v) {
|
if v := fake(t, f, "en_US."+c.path); !c.en.MatchString(v) {
|
||||||
@@ -61,11 +64,11 @@ func TestShippedDataCategories(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestShippedMiscCategories covers the locale-neutral data/misc folder: a proper
|
// TestShippedMiscCategories covers the locale-neutral misc module: a proper
|
||||||
// v4 UUID (version/variant nibbles fixed), a MAC address, and credit-card numbers
|
// v4 UUID (version/variant nibbles fixed), a MAC address, and credit-card numbers
|
||||||
// whose trailing {luhn()} check passes.
|
// whose trailing {luhn()} check passes.
|
||||||
func TestShippedMiscCategories(t *testing.T) {
|
func TestShippedMiscCategories(t *testing.T) {
|
||||||
f := newGenerator(t, "data/misc", WithSeed(1))
|
f := newBareModule(t, "misc", WithSeed(1))
|
||||||
v4 := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
|
v4 := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
|
||||||
mac := regexp.MustCompile(`^([0-9a-f]{2}:){5}[0-9a-f]{2}$`)
|
mac := regexp.MustCompile(`^([0-9a-f]{2}:){5}[0-9a-f]{2}$`)
|
||||||
objectid := regexp.MustCompile(`^[0-9a-f]{24}$`)
|
objectid := regexp.MustCompile(`^[0-9a-f]{24}$`)
|
||||||
@@ -93,7 +96,7 @@ func TestShippedMiscCategories(t *testing.T) {
|
|||||||
// codes follow their standard shape, dotted sub-paths resolve, emoji is a real
|
// codes follow their standard shape, dotted sub-paths resolve, emoji is a real
|
||||||
// glyph, and a coordinate parses to a point within geographic bounds.
|
// glyph, and a coordinate parses to a point within geographic bounds.
|
||||||
func TestShippedMiscReferenceData(t *testing.T) {
|
func TestShippedMiscReferenceData(t *testing.T) {
|
||||||
f := newGenerator(t, "data/misc", WithSeed(1))
|
f := newBareModule(t, "misc", WithSeed(1))
|
||||||
re := map[string]*regexp.Regexp{
|
re := map[string]*regexp.Regexp{
|
||||||
"currency": regexp.MustCompile(`^[A-Z]{3}$`),
|
"currency": regexp.MustCompile(`^[A-Z]{3}$`),
|
||||||
"currency.name": regexp.MustCompile(`\p{L}`),
|
"currency.name": regexp.MustCompile(`\p{L}`),
|
||||||
@@ -156,7 +159,7 @@ func TestShippedMiscReferenceData(t *testing.T) {
|
|||||||
// TestSwedishPersonNamesHaveNoTripleLetter pins an orthographic rule the shape
|
// TestSwedishPersonNamesHaveNoTripleLetter pins an orthographic rule the shape
|
||||||
// regexes miss: no generated name repeats a character three times over.
|
// regexes miss: no generated name repeats a character three times over.
|
||||||
func TestSwedishPersonNamesHaveNoTripleLetter(t *testing.T) {
|
func TestSwedishPersonNamesHaveNoTripleLetter(t *testing.T) {
|
||||||
f := newGenerator(t, "data", WithSeed(11))
|
f := shipped(t, WithSeed(11))
|
||||||
for _, path := range []string{"sv_SE.person", "sv_SE.person.last"} {
|
for _, path := range []string{"sv_SE.person", "sv_SE.person.last"} {
|
||||||
for i := 0; i < 20000; i++ {
|
for i := 0; i < 20000; i++ {
|
||||||
name := fake(t, f, path)
|
name := fake(t, f, path)
|
||||||
@@ -175,7 +178,7 @@ func TestSwedishPersonNamesHaveNoTripleLetter(t *testing.T) {
|
|||||||
// male), the trailing digit is a Luhn checksum over the other nine, and a
|
// male), the trailing digit is a Luhn checksum over the other nine, and a
|
||||||
// samordningsnummer is the same with 60 added to the day.
|
// samordningsnummer is the same with 60 added to the day.
|
||||||
func TestSwedishPersonnummer(t *testing.T) {
|
func TestSwedishPersonnummer(t *testing.T) {
|
||||||
sv := newGenerator(t, "data", WithSeed(1))
|
sv := shipped(t, WithSeed(1))
|
||||||
re := regexp.MustCompile(`^\d{6}-23[89]\d$`)
|
re := regexp.MustCompile(`^\d{6}-23[89]\d$`)
|
||||||
sawLongMonthEnd, birth := false, map[string]bool{}
|
sawLongMonthEnd, birth := false, map[string]bool{}
|
||||||
for i := 0; i < 2000; i++ {
|
for i := 0; i < 2000; i++ {
|
||||||
@@ -208,20 +211,13 @@ func TestSwedishPersonnummer(t *testing.T) {
|
|||||||
if !birth["238"] || !birth["239"] {
|
if !birth["238"] || !birth["239"] {
|
||||||
t.Fatalf("birth numbers drawn %v, want both 238 and 239", birth)
|
t.Fatalf("birth numbers drawn %v, want both 238 and 239", birth)
|
||||||
}
|
}
|
||||||
for i := 0; i < 200; i++ {
|
|
||||||
got := fakeTemplate(t, sv, `{/sv_SE.person.sex} {/sv_SE.personnummer}`)
|
|
||||||
sex, id, _ := strings.Cut(got, " ")
|
|
||||||
if want := map[string]string{"kvinna": "238", "man": "239"}[sex]; want == "" || !strings.Contains(id, "-"+want) {
|
|
||||||
t.Fatalf("%q: a person and a personnummer in one render disagree on sex", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestShippedUSTaxIds pins the SSA and IRS ranges: an SSN's area is 001-899 but
|
// TestShippedUSTaxIds pins the SSA and IRS ranges: an SSN's area is 001-899 but
|
||||||
// 666, its group 01-99 and its serial 0001-9999; an ITIN is 9XX-GG-XXXX with GG
|
// 666, its group 01-99 and its serial 0001-9999; an ITIN is 9XX-GG-XXXX with GG
|
||||||
// in 50-65, 70-88, 90-92 or 94-99.
|
// in 50-65, 70-88, 90-92 or 94-99.
|
||||||
func TestShippedUSTaxIds(t *testing.T) {
|
func TestShippedUSTaxIds(t *testing.T) {
|
||||||
f := newGenerator(t, "data", WithSeed(5))
|
f := shipped(t, WithSeed(5))
|
||||||
ssn := regexp.MustCompile(`^(\d{3})-(\d{2})-(\d{4})$`)
|
ssn := regexp.MustCompile(`^(\d{3})-(\d{2})-(\d{4})$`)
|
||||||
itin := regexp.MustCompile(`^9\d{2}-(5\d|6[0-5]|7\d|8[0-8]|9[0-24-9])-\d{4}$`)
|
itin := regexp.MustCompile(`^9\d{2}-(5\d|6[0-5]|7\d|8[0-8]|9[0-24-9])-\d{4}$`)
|
||||||
for i := 0; i < 2000; i++ {
|
for i := 0; i < 2000; i++ {
|
||||||
@@ -245,7 +241,7 @@ func TestShippedUSTaxIds(t *testing.T) {
|
|||||||
// TestShippedPersonNames pins the name tables: a sex pins its first names, a name
|
// TestShippedPersonNames pins the name tables: a sex pins its first names, a name
|
||||||
// both sexes carry appears under both, and the record's sex agrees with its name.
|
// both sexes carry appears under both, and the record's sex agrees with its name.
|
||||||
func TestShippedPersonNames(t *testing.T) {
|
func TestShippedPersonNames(t *testing.T) {
|
||||||
f := newGenerator(t, "data", WithSeed(9))
|
f := shipped(t, WithSeed(9))
|
||||||
for path, want := range map[string]string{
|
for path, want := range map[string]string{
|
||||||
"sv_SE.sex[f]": "kvinna",
|
"sv_SE.sex[f]": "kvinna",
|
||||||
"sv_SE.sex[man].code": "m",
|
"sv_SE.sex[man].code": "m",
|
||||||
@@ -273,24 +269,41 @@ func TestShippedPersonNames(t *testing.T) {
|
|||||||
seen := map[string]bool{}
|
seen := map[string]bool{}
|
||||||
for i := 0; i < 2000; i++ {
|
for i := 0; i < 2000; i++ {
|
||||||
seen[fake(t, f, locale+".sex[f].first-name")] = true
|
seen[fake(t, f, locale+".sex[f].first-name")] = true
|
||||||
first, sex := fake(t, f, locale+".person.first"), fake(t, f, locale+".person.sex")
|
got := fakeTemplate(t, f, `{/`+locale+`.person as p}{p.first}|{p.sex}`)
|
||||||
if first == "" || sex == "" {
|
first, sex, _ := strings.Cut(got, "|")
|
||||||
t.Fatalf("%s.person lacks a first name or a sex", locale)
|
fake(t, f, locale+".sex["+sex+"].first-name["+first+"]")
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if len(seen) < 300 || !seen["Anna"] && locale == "sv_SE" || !seen["Mary"] && locale == "en_US" {
|
if len(seen) < 300 || !seen["Anna"] && locale == "sv_SE" || !seen["Mary"] && locale == "en_US" {
|
||||||
t.Fatalf("%s female first names: %d distinct in 2000, want a weighted register", locale, len(seen))
|
t.Fatalf("%s female first names: %d distinct in 2000, want a weighted register", locale, len(seen))
|
||||||
}
|
}
|
||||||
got := fakeTemplate(t, f, `{/`+locale+`.sex.code}|{/`+locale+`.person.first}`)
|
}
|
||||||
code, first, _ := strings.Cut(got, "|")
|
}
|
||||||
if v := fake(t, f, locale+".sex["+code+"].first-name["+first+"]"); v != first {
|
|
||||||
t.Fatalf("%s: %q drawn as a %s name is not one", locale, first, code)
|
// TestShippedPersonIsOneNamedPick pins that a person's first name, sex and title come
|
||||||
|
// from one pick, and that two names bound to one person category are two people.
|
||||||
|
func TestShippedPersonIsOneNamedPick(t *testing.T) {
|
||||||
|
f := shipped(t, WithSeed(9))
|
||||||
|
for _, locale := range []string{"sv_SE", "en_US"} {
|
||||||
|
apart := false
|
||||||
|
for i := 0; i < 300; i++ {
|
||||||
|
got := fakeTemplate(t, f, `{/`+locale+`.person as a}{/`+locale+`.person as b}{a.first}|{a.sex}|{a.prefix}|{b.first}|{b.sex}`)
|
||||||
|
parts := strings.Split(got, "|")
|
||||||
|
for _, p := range [][2]string{{parts[0], parts[1]}, {parts[3], parts[4]}} {
|
||||||
|
fake(t, f, locale+".sex["+p[1]+"].first-name["+p[0]+"]")
|
||||||
|
}
|
||||||
|
if title := strings.TrimSpace(parts[2]); title != "" && locale == "en_US" {
|
||||||
|
fake(t, f, "en_US.sex["+parts[1]+"].title["+title+"]")
|
||||||
|
}
|
||||||
|
apart = apart || parts[0] != parts[3]
|
||||||
|
}
|
||||||
|
if !apart {
|
||||||
|
t.Errorf("%s: two names bound to person drew one first name in 300 renders, want two people", locale)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestShippedSwedishPhone(t *testing.T) {
|
func TestShippedSwedishPhone(t *testing.T) {
|
||||||
f := newGenerator(t, "data", WithSeed(11))
|
f := shipped(t, WithSeed(11))
|
||||||
re := regexp.MustCompile(`^0\d{1,2}-\d{3} \d{2} \d{2}$`)
|
re := regexp.MustCompile(`^0\d{1,2}-\d{3} \d{2} \d{2}$`)
|
||||||
for i := 0; i < 50; i++ {
|
for i := 0; i < 50; i++ {
|
||||||
if n := fake(t, f, "sv_SE.phone"); !re.MatchString(n) {
|
if n := fake(t, f, "sv_SE.phone"); !re.MatchString(n) {
|
||||||
@@ -300,7 +313,7 @@ func TestShippedSwedishPhone(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestShippedSwedishAddress(t *testing.T) {
|
func TestShippedSwedishAddress(t *testing.T) {
|
||||||
f := newGenerator(t, "data", WithSeed(3))
|
f := shipped(t, WithSeed(3))
|
||||||
digit := regexp.MustCompile(`\d`)
|
digit := regexp.MustCompile(`\d`)
|
||||||
for i := 0; i < 30; i++ {
|
for i := 0; i < 30; i++ {
|
||||||
a := fake(t, f, "sv_SE.address")
|
a := fake(t, f, "sv_SE.address")
|
||||||
@@ -317,8 +330,170 @@ func TestShippedSwedishAddress(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestShippedAddressIsOneNamedPick pins that an address's street, postal code,
|
||||||
|
// locality and region are one place in the geo tables, read through a name in a template or
|
||||||
|
// across a struct's tags, and that a reference beside the name is another address.
|
||||||
|
func TestShippedAddressIsOneNamedPick(t *testing.T) {
|
||||||
|
f := shipped(t, WithSeed(3))
|
||||||
|
for _, c := range []struct{ locale, country, template, largest string }{
|
||||||
|
{"sv_SE", "SE", "{a.street}|{a.postal-code}|{a.locality}|", "Stockholm"},
|
||||||
|
{"en_US", "US", "{a.street}|{a.postal-code}|{a.locality}|{a.region}", "New York"},
|
||||||
|
} {
|
||||||
|
places := geoPlaces(t, c.country)
|
||||||
|
apart, localities := false, map[string]int{}
|
||||||
|
for i := 0; i < 2000; i++ {
|
||||||
|
got := fakeTemplate(t, f, `{/`+c.locale+`.address as a}`+c.template+`|{/`+c.locale+`.address.street}`)
|
||||||
|
parts := strings.Split(got, "|")
|
||||||
|
if !places[strings.Join(parts[:4], "|")] {
|
||||||
|
t.Fatalf("%s.address: %q is no street, postal code, locality and region of one place", c.locale, got)
|
||||||
|
}
|
||||||
|
apart = apart || parts[0] != parts[4]
|
||||||
|
localities[parts[2]]++
|
||||||
|
}
|
||||||
|
if !apart {
|
||||||
|
t.Errorf("%s: a reference beside a name drew the name's street in 2000 renders, want two addresses", c.locale)
|
||||||
|
}
|
||||||
|
if c.country == "SE" {
|
||||||
|
var row struct {
|
||||||
|
Street string `fake:"{/sv_SE.address as a}{a.street}"`
|
||||||
|
PostalCode string `fake:"{a.postal-code}"`
|
||||||
|
Locality string `fake:"{a.locality}"`
|
||||||
|
}
|
||||||
|
for i := 0; i < 200; i++ {
|
||||||
|
if err := f.FakeStruct(&row); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := row.Street + "|" + row.PostalCode + "|" + row.Locality + "|"; !places[got] {
|
||||||
|
t.Fatalf("sv_SE.address: struct tags reading one name, %q, are no street, postal code and locality of one place", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for l, n := range localities {
|
||||||
|
if n > localities[c.largest] {
|
||||||
|
t.Errorf("%s.address drew %s %d times and %s %d, want the locality weighted by population", c.locale, l, n, c.largest, localities[c.largest])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 0644000 is Los Angeles.
|
||||||
|
func TestShippedPlacesSpreadAddresses(t *testing.T) {
|
||||||
|
for country, city := range map[string]struct {
|
||||||
|
key string
|
||||||
|
least int
|
||||||
|
}{"SE": {"Göteborg", 1500}, "US": {"0644000", 2500}} {
|
||||||
|
streets := 0
|
||||||
|
for _, r := range geoRows(t, country, "street") {
|
||||||
|
if r[1] == city.key {
|
||||||
|
streets++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if streets < city.least {
|
||||||
|
t.Errorf("geo.%s: %s ships %d streets, want at least %d", country, city.key, streets, city.least)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
header, _, _ := strings.Cut(geoTSV(t, "US", "locality"), "\n")
|
||||||
|
population := slices.Index(strings.Split(header, "\t"), "population")
|
||||||
|
if population < 0 {
|
||||||
|
t.Fatal("geo.US locality.tsv has no population column")
|
||||||
|
}
|
||||||
|
for _, r := range geoRows(t, "US", "locality") {
|
||||||
|
n, err := strconv.Atoi(r[population])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n < 2500 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Error("geo.US: every locality holds 2500 people or more, want places drawn down to villages")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestShippedCapitalsAndBigPlaces pins state capitals and places of 100,000 or more that a
|
||||||
|
// draw by population leaves out.
|
||||||
|
func TestShippedCapitalsAndBigPlaces(t *testing.T) {
|
||||||
|
state := map[string]string{}
|
||||||
|
for _, r := range geoRows(t, "US", "municipality") {
|
||||||
|
state[r[0]] = r[2]
|
||||||
|
}
|
||||||
|
shipped := map[string]bool{}
|
||||||
|
for _, r := range geoRows(t, "US", "locality") {
|
||||||
|
shipped[r[1]+", "+state[r[2]]] = true
|
||||||
|
}
|
||||||
|
for _, place := range []string{
|
||||||
|
"Albany, NY", "Augusta, ME", "Baton Rouge, LA", "Bismarck, ND", "Boise, ID", "Charleston, WV",
|
||||||
|
"Frankfort, KY", "Harrisburg, PA", "Helena, MT", "Juneau, AK", "Kansas City, KS", "Lafayette, LA",
|
||||||
|
"Lexington, KY", "Montpelier, VT", "New Orleans, LA", "Pierre, SD", "St. Paul, MN", "Stockton, CA",
|
||||||
|
"Ventura, CA",
|
||||||
|
} {
|
||||||
|
if !shipped[place] {
|
||||||
|
t.Errorf("geo.US: no locality %s, want every state capital and place of 100,000 or more", place)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestShippedSwedishStreetsAreStreetNames(t *testing.T) {
|
||||||
|
refused := map[string]bool{
|
||||||
|
"Färjeled Henriksdalsbryggan - Barnängsbryggan": true, "HÄLLSÄTTERSLEDEN": true, "Kärrs VSF": true,
|
||||||
|
"Länsväg 694": true, "NN": true, "Risövägen,Dejstolvägen,Runvägen": true, "tornslingan": true,
|
||||||
|
"Öresundsvägen, Bondevägen": true,
|
||||||
|
}
|
||||||
|
for _, r := range geoRows(t, "SE", "street") {
|
||||||
|
if refused[r[0]] {
|
||||||
|
t.Errorf("geo.SE: street %q in %s, want no road number, ferry route, label, placeholder, two names or miscased name", r[0], r[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func geoTSV(t *testing.T, country, table string) string {
|
||||||
|
b, err := os.ReadFile("data/geo/" + country + "/" + table + ".tsv")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(string(b))
|
||||||
|
}
|
||||||
|
|
||||||
|
func geoRows(t *testing.T, country, table string) [][]string {
|
||||||
|
var out [][]string
|
||||||
|
for _, line := range strings.Split(geoTSV(t, country, table), "\n")[1:] {
|
||||||
|
out = append(out, strings.Split(line, "\t"))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// geoPlaces reads a country's geo tables into every "street|postal-code|locality|region"
|
||||||
|
// one locality holds, the region empty for SE, whose address carries none.
|
||||||
|
func geoPlaces(t *testing.T, country string) map[string]bool {
|
||||||
|
rows := func(table string) [][]string { return geoRows(t, country, table) }
|
||||||
|
name, region := map[string]string{}, map[string]string{}
|
||||||
|
if country == "US" {
|
||||||
|
county := map[string]string{}
|
||||||
|
for _, r := range rows("municipality") {
|
||||||
|
county[r[0]] = r[2]
|
||||||
|
}
|
||||||
|
for _, r := range rows("locality") {
|
||||||
|
name[r[0]], region[r[0]] = r[1], county[r[2]]
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for _, r := range rows("locality") {
|
||||||
|
name[r[0]] = r[0]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
codes := map[string][]string{}
|
||||||
|
for _, r := range rows("postal-code") {
|
||||||
|
codes[r[1]] = append(codes[r[1]], r[0])
|
||||||
|
}
|
||||||
|
places := map[string]bool{}
|
||||||
|
for _, r := range rows("street") {
|
||||||
|
for _, code := range codes[r[1]] {
|
||||||
|
places[strings.Join([]string{r[0], code, name[r[1]], region[r[1]]}, "|")] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return places
|
||||||
|
}
|
||||||
|
|
||||||
func TestShippedPersonHasParts(t *testing.T) {
|
func TestShippedPersonHasParts(t *testing.T) {
|
||||||
f := newGenerator(t, "data", WithSeed(7))
|
f := shipped(t, WithSeed(7))
|
||||||
for _, path := range []string{"sv_SE.person", "en_US.person"} {
|
for _, path := range []string{"sv_SE.person", "en_US.person"} {
|
||||||
for i := 0; i < 30; i++ {
|
for i := 0; i < 30; i++ {
|
||||||
if name := fake(t, f, path); len(name) < 3 || !regexp.MustCompile(`\S \S`).MatchString(name) {
|
if name := fake(t, f, path); len(name) < 3 || !regexp.MustCompile(`\S \S`).MatchString(name) {
|
||||||
@@ -328,20 +503,50 @@ func TestShippedPersonHasParts(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NANPA's central office code records mark N11, 555, 950, 958, 959 and 976 unassignable.
|
||||||
func TestShippedUSPhone(t *testing.T) {
|
func TestShippedUSPhone(t *testing.T) {
|
||||||
f := newGenerator(t, "data", WithSeed(11))
|
f := shipped(t, WithSeed(11))
|
||||||
re := regexp.MustCompile(`^(\(\d{3}\) \d{3}-\d{4}|\d{3}-\d{3}-\d{4})$`)
|
re := regexp.MustCompile(`^(?:\(\d{3}\) |\d{3}-)([2-9]\d{2})-\d{4}$`)
|
||||||
for i := 0; i < 50; i++ {
|
unassignable := map[string]bool{"555": true, "950": true, "958": true, "959": true, "976": true}
|
||||||
if n := fake(t, f, "en_US.phone"); !re.MatchString(n) {
|
for i := 0; i < 2000; i++ {
|
||||||
|
n := fake(t, f, "en_US.phone")
|
||||||
|
m := re.FindStringSubmatch(n)
|
||||||
|
if m == nil {
|
||||||
t.Fatalf("phone %q does not match %s", n, re)
|
t.Fatalf("phone %q does not match %s", n, re)
|
||||||
}
|
}
|
||||||
|
if exchange := m[1]; exchange[1:] == "11" || unassignable[exchange] {
|
||||||
|
t.Fatalf("phone %q has the unassignable exchange %s", n, exchange)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestShippedNamespacedTree loads the whole data/ tree (not a single locale) and
|
func TestShippedTableCellsAreTrimmed(t *testing.T) {
|
||||||
// reaches each locale through its folder segment: data/sv_SE/person -> sv_SE.person.
|
err := fs.WalkDir(os.DirFS("data"), ".", func(path string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil || !strings.HasSuffix(path, ".tsv") {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
b, err := fs.ReadFile(os.DirFS("data"), path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for i, line := range strings.Split(string(b), "\n") {
|
||||||
|
for _, cell := range strings.Split(line, "\t") {
|
||||||
|
if cell != strings.TrimSpace(cell) {
|
||||||
|
t.Errorf("data/%s:%d: cell %q is padded", path, i+1, cell)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestShippedNamespacedTree loads every shipped module and reaches each locale through its
|
||||||
|
// namespace: data/sv_SE/person.json, mounted under sv_SE, is sv_SE.person.
|
||||||
func TestShippedNamespacedTree(t *testing.T) {
|
func TestShippedNamespacedTree(t *testing.T) {
|
||||||
f := newGenerator(t, "data", WithSeed(1))
|
f := shipped(t, WithSeed(1))
|
||||||
for _, path := range []string{"sv_SE.person", "en_US.person", "sv_SE.address.locality"} {
|
for _, path := range []string{"sv_SE.person", "en_US.person", "sv_SE.address.locality"} {
|
||||||
if got := fake(t, f, path); got == "" {
|
if got := fake(t, f, path); got == "" {
|
||||||
t.Fatalf("Fake(%q) returned empty", path)
|
t.Fatalf("Fake(%q) returned empty", path)
|
||||||
@@ -385,7 +590,7 @@ func luhnValid(s string) bool {
|
|||||||
var swedishName = regexp.MustCompile(`^\p{L}+([ -]\p{L}+)*$`)
|
var swedishName = regexp.MustCompile(`^\p{L}+([ -]\p{L}+)*$`)
|
||||||
|
|
||||||
func TestShippedStreetIsARegisteredName(t *testing.T) {
|
func TestShippedStreetIsARegisteredName(t *testing.T) {
|
||||||
f := newGenerator(t, "data", WithSeed(5))
|
f := shipped(t, WithSeed(5))
|
||||||
street := regexp.MustCompile(`^\p{L}[\p{L}\d:.-]*([ -][\p{L}\d:.-]+)*$`)
|
street := regexp.MustCompile(`^\p{L}[\p{L}\d:.-]*([ -][\p{L}\d:.-]+)*$`)
|
||||||
for i := 0; i < 300; i++ {
|
for i := 0; i < 300; i++ {
|
||||||
if s := fake(t, f, "sv_SE.address.street"); !street.MatchString(s) {
|
if s := fake(t, f, "sv_SE.address.street"); !street.MatchString(s) {
|
||||||
@@ -397,7 +602,7 @@ func TestShippedStreetIsARegisteredName(t *testing.T) {
|
|||||||
func TestShippedLastNameComposition(t *testing.T) {
|
func TestShippedLastNameComposition(t *testing.T) {
|
||||||
// last is a choice of patronymic {first}sson templates, compound
|
// last is a choice of patronymic {first}sson templates, compound
|
||||||
// {first}{last} templates and literal surnames.
|
// {first}{last} templates and literal surnames.
|
||||||
f := newGenerator(t, "data", WithSeed(6))
|
f := shipped(t, WithSeed(6))
|
||||||
for i := 0; i < 300; i++ {
|
for i := 0; i < 300; i++ {
|
||||||
if s := fake(t, f, "sv_SE.person.last"); !swedishName.MatchString(s) {
|
if s := fake(t, f, "sv_SE.person.last"); !swedishName.MatchString(s) {
|
||||||
t.Fatalf("last name %q is not a Swedish surname", s)
|
t.Fatalf("last name %q is not a Swedish surname", s)
|
||||||
@@ -407,7 +612,7 @@ func TestShippedLastNameComposition(t *testing.T) {
|
|||||||
|
|
||||||
func TestShippedStreetNumberFormats(t *testing.T) {
|
func TestShippedStreetNumberFormats(t *testing.T) {
|
||||||
// Reachable via a hyphenated path; covers all five weighted number variants.
|
// Reachable via a hyphenated path; covers all five weighted number variants.
|
||||||
f := newGenerator(t, "data", WithSeed(8))
|
f := shipped(t, WithSeed(8))
|
||||||
re := regexp.MustCompile(`^[1-9]\d{0,2}[A-Z]?$`)
|
re := regexp.MustCompile(`^[1-9]\d{0,2}[A-Z]?$`)
|
||||||
for i := 0; i < 300; i++ {
|
for i := 0; i < 300; i++ {
|
||||||
if n := fake(t, f, "sv_SE.address.street-number"); !re.MatchString(n) {
|
if n := fake(t, f, "sv_SE.address.street-number"); !re.MatchString(n) {
|
||||||
@@ -419,11 +624,11 @@ func TestShippedStreetNumberFormats(t *testing.T) {
|
|||||||
// TestShippedUSTitleAgreesWithSex pins that one person's title and sex agree, and
|
// TestShippedUSTitleAgreesWithSex pins that one person's title and sex agree, and
|
||||||
// that the everyday titles are reachable.
|
// that the everyday titles are reachable.
|
||||||
func TestShippedUSTitleAgreesWithSex(t *testing.T) {
|
func TestShippedUSTitleAgreesWithSex(t *testing.T) {
|
||||||
f := newGenerator(t, "data", WithSeed(4))
|
f := shipped(t, WithSeed(4))
|
||||||
female := map[string]bool{"Miss": true, "Mrs": true, "Ms": true}
|
female := map[string]bool{"Miss": true, "Mrs": true, "Ms": true}
|
||||||
seen := map[string]bool{}
|
seen := map[string]bool{}
|
||||||
for i := 0; i < 3000; i++ {
|
for i := 0; i < 3000; i++ {
|
||||||
got := fakeTemplate(t, f, `{/en_US.person.prefix}|{/en_US.person.sex}`)
|
got := fakeTemplate(t, f, `{/en_US.person as p}{p.prefix}|{p.sex}`)
|
||||||
title, sex, _ := strings.Cut(got, "|")
|
title, sex, _ := strings.Cut(got, "|")
|
||||||
title = strings.TrimSpace(title)
|
title = strings.TrimSpace(title)
|
||||||
seen[title] = true
|
seen[title] = true
|
||||||
|
|||||||
+161
-113
@@ -3,97 +3,83 @@ package fejkdata
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata/internal/datatype"
|
||||||
|
"github.com/larvit/fejkdata/internal/grammar"
|
||||||
|
"github.com/larvit/fejkdata/internal/invariant"
|
||||||
|
"github.com/larvit/fejkdata/internal/jsonvalue"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DataType is what a record column holds, which decides how a record writes its value.
|
// DataType is what a record column holds, which decides how a record writes its value. It
|
||||||
type DataType int
|
// prints as data spells it: string, integer, number or boolean.
|
||||||
|
type DataType = datatype.DataType
|
||||||
|
|
||||||
// The datatypes a column declares with "datatype"; a column without one is a string.
|
// The datatypes a column declares with "datatype"; a column without one is a string.
|
||||||
const (
|
const (
|
||||||
DataTypeString DataType = iota
|
DataTypeString DataType = datatype.String
|
||||||
DataTypeInteger
|
DataTypeInteger DataType = datatype.Integer
|
||||||
DataTypeNumber
|
DataTypeNumber DataType = datatype.Number
|
||||||
DataTypeBoolean
|
DataTypeBoolean DataType = datatype.Boolean
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
// datatypeOf reads a template's "datatype", nil where it carries none or sits outside a column.
|
||||||
dataTypeNames = [...]string{"string", "integer", "number", "boolean"}
|
func datatypeOf(m map[string]any, pos position) (*DataType, error) {
|
||||||
dataTypeNouns = [...]string{"text", "an integer", "a number", "a boolean"}
|
|
||||||
)
|
|
||||||
|
|
||||||
// String is the datatype as data spells it.
|
|
||||||
func (d DataType) String() string {
|
|
||||||
if d < 0 || int(d) >= len(dataTypeNames) {
|
|
||||||
return fmt.Sprintf("DataType(%d)", int(d))
|
|
||||||
}
|
|
||||||
return dataTypeNames[d]
|
|
||||||
}
|
|
||||||
|
|
||||||
// position is where a JSON value sits, which decides whether it may carry a datatype or
|
|
||||||
// be null.
|
|
||||||
type position int
|
|
||||||
|
|
||||||
const (
|
|
||||||
inFormat position = iota // rendered by a format, so neither
|
|
||||||
atTop // a category or an inline template, whose fields may be columns
|
|
||||||
inColumn // a column, or a choice item standing in for one
|
|
||||||
)
|
|
||||||
|
|
||||||
// datatypeOf reads a template's "datatype" (default DataTypeString).
|
|
||||||
func datatypeOf(m map[string]any, pos position) (DataType, error) {
|
|
||||||
v, ok := m["datatype"]
|
v, ok := m["datatype"]
|
||||||
if !ok {
|
if !ok {
|
||||||
return DataTypeString, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
name, ok := v.(string)
|
name, ok := v.(string)
|
||||||
if !ok {
|
if !ok {
|
||||||
return 0, fmt.Errorf("datatype must be a string, got %T", v)
|
return nil, fmt.Errorf("datatype must be a string, not %s", jsonvalue.Kind(v))
|
||||||
}
|
}
|
||||||
if name == DataTypeString.String() {
|
for d := DataTypeString; d < datatype.Count; d++ {
|
||||||
return 0, fmt.Errorf("datatype %q is the default, so it has no effect; drop it", name)
|
|
||||||
}
|
|
||||||
for d := DataTypeInteger; d <= DataTypeBoolean; d++ {
|
|
||||||
if name != d.String() {
|
if name != d.String() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if pos != inColumn {
|
if !pos.column() {
|
||||||
return 0, errors.New("datatype only types a record column — a field of the top-level template — so it has no effect here")
|
return nil, nil
|
||||||
}
|
}
|
||||||
return d, nil
|
return &d, nil
|
||||||
}
|
}
|
||||||
return 0, fmt.Errorf(`datatype takes "integer", "number" or "boolean", got %q`, name)
|
return nil, fmt.Errorf(`datatype takes "string", "integer", "number" or "boolean", got %q`, name)
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkColumns rejects a record column whose items hold different datatypes, checking a column
|
// checkColumns rejects a record column whose items hold different datatypes, checking a column
|
||||||
// after the columns its items read, so a column read is named before its readers.
|
// after the columns its items read, so a column read is named before its readers.
|
||||||
func checkColumns(s nodeScope) error {
|
func checkColumns(s nodeSet) error {
|
||||||
checked := map[node]bool{}
|
checked := map[node]bool{}
|
||||||
var check func(path, name string, column node) error
|
var check func(label, name string, column node) error
|
||||||
check = func(path, name string, column node) error {
|
check = func(label, name string, column node) error {
|
||||||
if checked[column] {
|
if checked[column] {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
checked[column] = true
|
checked[column] = true
|
||||||
items, _ := columnItems(column)
|
items, _ := columnItems(column)
|
||||||
for _, it := range items {
|
for _, it := range items {
|
||||||
if r := it.readsColumn; r != nil {
|
if r := it.readsColumn; r != nil && r.category != "" {
|
||||||
if err := check(r.a.key[1:], r.a.tail[0], r.column); err != nil {
|
if err := check(r.category, r.field, r.column); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if _, err := columnDatatype(column); err != nil {
|
if len(items) == 0 {
|
||||||
return fmt.Errorf("%s: field %q: %w", path, name, err)
|
return nil
|
||||||
|
}
|
||||||
|
first := columnDatatype(column)
|
||||||
|
for _, it := range items[1:] {
|
||||||
|
if d := itemDatatype(it); d != first {
|
||||||
|
return fmt.Errorf("%s: field %q: %w", label, name, disagreement(items[0], first, it, d))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return s(func(path string, n node) error {
|
return s(func(label string, n node) error {
|
||||||
t, ok := n.(*template)
|
t, ok := n.(*template)
|
||||||
if !ok || !t.record {
|
if !ok || !t.isRecord {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
for _, name := range recordColumns(t) {
|
for _, name := range sortedNames(t.fields) {
|
||||||
if err := check(path, name, t.fields[name]); err != nil {
|
if err := check(label, name, t.fields[name]); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -101,41 +87,25 @@ func checkColumns(s nodeScope) error {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// columnDatatype is the datatype a column's items hold. They must agree, since a
|
// columnDatatype is the datatype a column holds, which its first item decides: checkColumns refuses
|
||||||
// column holds one; a column only ever null is a string.
|
// a column whose items disagree. A table's column, like one only ever null, has no item and is a string.
|
||||||
func columnDatatype(n node) (DataType, error) {
|
func columnDatatype(n node) DataType {
|
||||||
items, _ := columnItems(n)
|
items, _ := columnItems(n)
|
||||||
if len(items) == 0 {
|
if len(items) == 0 {
|
||||||
return DataTypeString, nil
|
return DataTypeString
|
||||||
}
|
}
|
||||||
first := itemDatatype(items[0])
|
return itemDatatype(items[0])
|
||||||
for _, t := range items[1:] {
|
|
||||||
if d := itemDatatype(t); d != first {
|
|
||||||
return first, disagreement(items[0], first, t, d)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return first, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// itemDatatype is the datatype a column item declares, else that of the column it is.
|
// itemDatatype is the datatype a column item declares, else that of the column it reads.
|
||||||
func itemDatatype(t *template) DataType {
|
func itemDatatype(t *template) DataType {
|
||||||
if t.datatype != DataTypeString {
|
switch {
|
||||||
return t.datatype
|
case t.datatype != nil:
|
||||||
|
return *t.datatype
|
||||||
|
case t.readsColumn != nil:
|
||||||
|
return columnDatatype(t.readsColumn.column)
|
||||||
}
|
}
|
||||||
return readDatatype(t)
|
return DataTypeString
|
||||||
}
|
|
||||||
|
|
||||||
// readDatatype is the datatype of the column t is, reading it by one reference alone; a string
|
|
||||||
// when t reads none.
|
|
||||||
func readDatatype(t *template) DataType {
|
|
||||||
if t.readsColumn == nil {
|
|
||||||
return DataTypeString
|
|
||||||
}
|
|
||||||
items, _ := columnItems(t.readsColumn.column)
|
|
||||||
if len(items) == 0 {
|
|
||||||
return DataTypeString
|
|
||||||
}
|
|
||||||
return itemDatatype(items[0]) // checkColumns refuses that column where its items disagree
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// columnItems is a column's template items, its choices unwrapped, and whether one is null.
|
// columnItems is a column's template items, its choices unwrapped, and whether one is null.
|
||||||
@@ -149,45 +119,91 @@ func columnItems(n node) (items []*template, nullable bool) {
|
|||||||
}
|
}
|
||||||
case *template:
|
case *template:
|
||||||
items = append(items, n)
|
items = append(items, n)
|
||||||
case *null:
|
case *nullItem:
|
||||||
nullable = true
|
nullable = true
|
||||||
|
case *tableColumn:
|
||||||
|
default:
|
||||||
|
panic(invariant.Broken("columnItems has no case for node %T", n))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
collect(n)
|
collect(n)
|
||||||
return items, nullable
|
return items, nullable
|
||||||
}
|
}
|
||||||
|
|
||||||
// disagreement names the fix for two items of one column holding different datatypes.
|
// itemKind is how a column item comes by its datatype, ranked in this order: none, read from a typed column, or declared.
|
||||||
func disagreement(a *template, da DataType, b *template, db DataType) error {
|
type itemKind int
|
||||||
bare, typed, want := b, a, da
|
|
||||||
if da == DataTypeString {
|
const (
|
||||||
bare, typed, want = a, b, db
|
kindText itemKind = iota
|
||||||
}
|
kindReads
|
||||||
|
kindDeclares
|
||||||
|
)
|
||||||
|
|
||||||
|
func kindOf(t *template) itemKind {
|
||||||
switch {
|
switch {
|
||||||
case da != DataTypeString && db != DataTypeString:
|
case t.datatype != nil:
|
||||||
return bothTyped(a, da, b, db)
|
return kindDeclares
|
||||||
case typed.datatype == DataTypeString && (&valueProof{}).columnItem(bare).not[want] != "":
|
case itemDatatype(t) != DataTypeString:
|
||||||
return fmt.Errorf(`item %q is not %s, the datatype item %q takes from the column it reads; to read that column as text, %s`, bare.format, dataTypeNouns[want], typed.format, asText(typed))
|
return kindReads
|
||||||
case bare.fromString: // an object may carry a weight, which this spelling would drop
|
|
||||||
return fmt.Errorf(`item %q declares no datatype, and a column holds one; write it as {"format":%q,"datatype":%q}`, bare.format, bare.format, want)
|
|
||||||
}
|
}
|
||||||
return fmt.Errorf(`item %q declares no datatype beside one holding %s; a column holds one, so give it "datatype": %q`, bare.format, want, want)
|
return kindText
|
||||||
}
|
}
|
||||||
|
|
||||||
// bothTyped names the fix for two items holding different datatypes: the one taking its datatype
|
// disagreement names the fix for items a and b of one column holding different datatypes, fixing
|
||||||
// from the column it reads is typed as the other where its values prove it, else read as text.
|
// the one that declares least, a when both declare alike.
|
||||||
func bothTyped(a *template, da DataType, b *template, db DataType) error {
|
func disagreement(a *template, da DataType, b *template, db DataType) error {
|
||||||
read, other := a, db
|
c := clash{fix: a, other: b, want: db, held: fmt.Sprintf("its items hold %s and %s; a column holds one datatype", da, db)}
|
||||||
if a.datatype != DataTypeString {
|
if kindOf(b) < kindOf(a) {
|
||||||
read, other = b, da
|
c.fix, c.other, c.want = b, a, da
|
||||||
}
|
}
|
||||||
switch {
|
message, ok := disagreements[[2]itemKind{kindOf(c.fix), kindOf(c.other)}]
|
||||||
case read.datatype != DataTypeString:
|
if !ok {
|
||||||
return fmt.Errorf("its items hold %s and %s; a column holds one datatype", da, db)
|
panic(invariant.Broken("disagreement has no message for item kinds %d and %d; two text items hold one datatype, so they never disagree", kindOf(c.fix), kindOf(c.other)))
|
||||||
case (&valueProof{}).columnItem(read).not[other] == "":
|
|
||||||
return fmt.Errorf("its items hold %s and %s; a column holds one datatype, so %s", da, db, typedAs(read, other))
|
|
||||||
}
|
}
|
||||||
return fmt.Errorf("its items hold %s and %s; a column holds one datatype, so to read %q as text, %s", da, db, read.format, asText(read))
|
return message(c)
|
||||||
|
}
|
||||||
|
|
||||||
|
// clash is two items of one column holding different datatypes: other holds want; held says
|
||||||
|
// what each holds.
|
||||||
|
type clash struct {
|
||||||
|
fix, other *template
|
||||||
|
want DataType
|
||||||
|
held string
|
||||||
|
}
|
||||||
|
|
||||||
|
// disagreements is a clash's message by the kinds of its items, fix's first.
|
||||||
|
var disagreements = map[[2]itemKind]func(clash) error{
|
||||||
|
{kindDeclares, kindDeclares}: clash.bothDeclare,
|
||||||
|
{kindReads, kindReads}: clash.retypeRead,
|
||||||
|
{kindReads, kindDeclares}: clash.retypeRead,
|
||||||
|
{kindText, kindReads}: clash.textBesideRead,
|
||||||
|
{kindText, kindDeclares}: clash.declareText,
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c clash) bothDeclare() error { return errors.New(c.held) }
|
||||||
|
|
||||||
|
// fits reports whether fix may hold want.
|
||||||
|
func (c clash) fits() bool { return (&valueProof{}).proveColumnItem(c.fix).Not[c.want] == "" }
|
||||||
|
|
||||||
|
func (c clash) retypeRead() error {
|
||||||
|
if c.fits() {
|
||||||
|
return fmt.Errorf("%s, so %s", c.held, typedAs(c.fix, c.want))
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%s, so to read %q as text, %s", c.held, c.fix.format, typedAs(c.fix, DataTypeString))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c clash) textBesideRead() error {
|
||||||
|
if !c.fits() {
|
||||||
|
return fmt.Errorf(`item %q is not %s, the datatype item %q takes from the column it reads; to read that column as text, %s`, c.fix.format, datatype.Noun(c.want), c.other.format, typedAs(c.other, DataTypeString))
|
||||||
|
}
|
||||||
|
return c.declareText()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c clash) declareText() error {
|
||||||
|
if c.fix.fromString { // an object may carry a weight, which this spelling would drop
|
||||||
|
return fmt.Errorf(`item %q declares no datatype, and a column holds one; write it as {"format":%q,"datatype":%q}`, c.fix.format, c.fix.format, c.want)
|
||||||
|
}
|
||||||
|
return fmt.Errorf(`item %q declares no datatype beside one holding %s; a column holds one, so give it "datatype": %q`, c.fix.format, c.want, c.want)
|
||||||
}
|
}
|
||||||
|
|
||||||
// typedAs names the spelling giving a column-read item datatype d, keeping the other keys an object
|
// typedAs names the spelling giving a column-read item datatype d, keeping the other keys an object
|
||||||
@@ -199,11 +215,43 @@ func typedAs(t *template, d DataType) string {
|
|||||||
return fmt.Sprintf(`give %q "datatype": %q`, t.format, d)
|
return fmt.Sprintf(`give %q "datatype": %q`, t.format, d)
|
||||||
}
|
}
|
||||||
|
|
||||||
// asText names the spelling that reads the column a column-read item reads as text, keeping the
|
// columnRead is a record's column read by a format that only reads one reference or name,
|
||||||
// other keys an object item carries.
|
// which is the column: it takes the column's datatype and null. category and field name it;
|
||||||
func asText(t *template) string {
|
// category is "" for a column of the reading template's own record, which checkColumns reaches
|
||||||
if t.fromString {
|
// on its own.
|
||||||
return fmt.Sprintf(`write {"format":"{text}","text":%q}`, t.format)
|
type columnRead struct {
|
||||||
}
|
a arm
|
||||||
return fmt.Sprintf(`set its "format" to "{text}" and add "text": %q`, t.format)
|
category, field string
|
||||||
|
column node
|
||||||
|
}
|
||||||
|
|
||||||
|
// columnReadOf is the record's column t's format reads, where the format only reads one reference
|
||||||
|
// or name, and nil where it does not or t declares a string, which reads the column as text.
|
||||||
|
func columnReadOf(t *template, targets map[*nameBinding]nameTarget) *columnRead {
|
||||||
|
ops := t.compiled.ops
|
||||||
|
if t.datatype != nil && *t.datatype == DataTypeString || t.repeat != 1 || len(ops) != 1 || ops[0].Kind != grammar.PathRead || len(ops[0].arms) != 1 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
a := ops[0].arms[0]
|
||||||
|
var start node
|
||||||
|
var category string
|
||||||
|
var tail []string
|
||||||
|
switch b := a.named; {
|
||||||
|
case a.kind == namedRead && b.bindsField():
|
||||||
|
// {f as n}{n}: column f of the record binding n, in t's own category.
|
||||||
|
start, tail = b.binder, append(append([]string{splitArm(b.ref, nil).head}, targets[b].tail...), a.tail...)
|
||||||
|
case a.kind == namedRead:
|
||||||
|
// {/c as n}{n.x} or {/c.x as n}{n}: column x of category c.
|
||||||
|
start, category = targets[b].start, categoryOf(b.binder.refs.byName[b.ref].head)
|
||||||
|
tail = append(targets[b].tail[:len(targets[b].tail):len(targets[b].tail)], a.tail...)
|
||||||
|
case grammar.IsRef(a.head):
|
||||||
|
// {/c.x}: column x of category c.
|
||||||
|
start, category, tail = t.startOf(a.head), categoryOf(a.head), a.tail
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if target, isTemplate := start.(*template); isTemplate && target.isRecord && len(tail) == 1 {
|
||||||
|
return &columnRead{a: a, category: category, field: tail[0], column: target.fields[tail[0]]}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
+70
-24
@@ -9,7 +9,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) {
|
func TestDatatypeSitsOnlyInAColumn(t *testing.T) {
|
||||||
for _, src := range []string{
|
for _, src := range []string{
|
||||||
`{"format":"","age":{"format":"{int(18,99)}","datatype":"integer"}}`,
|
`{"format":"","age":{"format":"{int(18,99)}","datatype":"integer"}}`,
|
||||||
`{"format":"","n":{"format":"42","datatype":"integer"}}`,
|
`{"format":"","n":{"format":"42","datatype":"integer"}}`,
|
||||||
@@ -18,21 +18,15 @@ func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) {
|
|||||||
`{"format":"","age":[null,{"format":"{int(18,99)}","datatype":"integer","weight":9}]}`,
|
`{"format":"","age":[null,{"format":"{int(18,99)}","datatype":"integer","weight":9}]}`,
|
||||||
`{"format":"","pick":[[null,"a"],"b"]}`,
|
`{"format":"","pick":[[null,"a"],"b"]}`,
|
||||||
} {
|
} {
|
||||||
if _, err := compile(parse(t, src)); err != nil {
|
if _, err := resolved(t, src); err != nil {
|
||||||
t.Errorf("compile(%s) = %v, want a column to take a datatype and null", src, err)
|
t.Errorf("compile(%s) = %v, want a column to take a datatype and null", src, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for src, want := range map[string]string{
|
for src, want := range map[string]string{
|
||||||
`{"format":"","n":{"format":"1","datatype":"int"}}`: `datatype takes "integer", "number" or "boolean", got "int"`,
|
`{"format":"","n":{"format":"1","datatype":"int"}}`: `datatype takes "string", "integer", "number" or "boolean", got "int"`,
|
||||||
`{"format":"","n":{"format":"1","datatype":1}}`: "datatype must be a string",
|
`{"format":"","n":{"format":"1","datatype":1}}`: "datatype must be a string, not a number",
|
||||||
`{"format":"{int(1,9)}","datatype":"integer"}`: "datatype only types a record column",
|
|
||||||
`[{"format":"1","datatype":"integer"},"x"]`: "datatype only types a record column",
|
|
||||||
`{"format":"{p}","p":{"format":"{n}","n":{"format":"1","datatype":"integer"}}}`: "datatype only types a record column",
|
|
||||||
`{"format":"{n}","repeat":2,"n":{"format":"1","datatype":"integer"}}`: "datatype only types a record column",
|
|
||||||
`null`: `so write ""`,
|
|
||||||
`{"format":"{p}","p":{"format":"{x}","x":[null,"a"]}}`: `so write ""`,
|
|
||||||
} {
|
} {
|
||||||
if _, err := compile(parse(t, src)); err == nil || !strings.Contains(err.Error(), want) {
|
if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
|
||||||
t.Errorf("compile(%s) = %v, want an error containing %q", src, err, want)
|
t.Errorf("compile(%s) = %v, want an error containing %q", src, err, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -41,27 +35,30 @@ func TestDatatypeAndNullSitOnlyInAColumn(t *testing.T) {
|
|||||||
func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
|
func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
|
||||||
tree := map[string]string{
|
tree := map[string]string{
|
||||||
"cat": `[{"format":"{code}","code":"200"},{"format":"{code}","code":"2x","note":"n"}]`,
|
"cat": `[{"format":"{code}","code":"200"},{"format":"{code}","code":"2x","note":"n"}]`,
|
||||||
"src": `{"format":"","code":[null,"200","2x"],"flag":{"format":"{b}","b":["true","false"],"datatype":"boolean"},"score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`,
|
"src": `{"format":"","code":[null,"200","2x"],"flag":{"format":"{b}","b":["true","false"],"datatype":"boolean"},"ratio":{"format":"{float(0,9,0)}","datatype":"number"},"score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`,
|
||||||
}
|
}
|
||||||
for _, c := range []struct{ name, column, want string }{
|
for _, c := range []struct{ name, column, want string }{
|
||||||
{"an item beside a typed one", `[{"format":"1","datatype":"integer"},"x"]`, `write it as {"format":"x","datatype":"integer"}`},
|
{"an item beside a typed one", `[{"format":"1","datatype":"integer"},"x"]`, `write it as {"format":"x","datatype":"integer"}`},
|
||||||
{"a weighted item beside a typed one", `[{"format":"1","datatype":"integer"},{"format":"2","weight":3}]`, `give it "datatype": "integer"`},
|
{"a weighted item beside a typed one", `[{"format":"1","datatype":"integer"},{"format":"2","weight":3}]`, `give it "datatype": "integer"`},
|
||||||
{"items of two datatypes", `[{"format":"1","datatype":"integer"},{"format":"true","datatype":"boolean"}]`, "a column holds one datatype"},
|
|
||||||
{"an item beside a typed column it reads", `["{/src.score}","5"]`, `write it as {"format":"5","datatype":"integer"}`},
|
{"an item beside a typed column it reads", `["{/src.score}","5"]`, `write it as {"format":"5","datatype":"integer"}`},
|
||||||
{"a typed item beside a string column it reads", `["{/src.code}",{"format":"1","datatype":"integer"}]`, `write it as {"format":"{/src.code}","datatype":"integer"}`},
|
{"a typed item beside a string column it reads", `["{/src.code}",{"format":"1","datatype":"integer"}]`, `write it as {"format":"{/src.code}","datatype":"integer"}`},
|
||||||
{"text beside a typed column it reads", `["{/src.score}","n/a"]`, `to read that column as text, write {"format":"{text}","text":"{/src.score}"}`},
|
{"text beside a typed column it reads", `["{/src.score}","n/a"]`, `to read that column as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
|
||||||
{"a datatype over a typed column", `{"format":"{/src.score}","datatype":"integer"}`, `{/src.score} takes datatype integer from the column it reads; drop "datatype"`},
|
|
||||||
{"a datatype a typed column's values reject", `{"format":"{/src.score}","datatype":"boolean"}`, "{int(1,9)} prints an integer, not a boolean"},
|
{"a datatype a typed column's values reject", `{"format":"{/src.score}","datatype":"boolean"}`, "{int(1,9)} prints an integer, not a boolean"},
|
||||||
{"two typed columns it reads", `["{/src.score}","{/src.flag}"]`, `so to read "{/src.score}" as text, write {"format":"{text}","text":"{/src.score}"}`},
|
{"two typed columns it reads", `["{/src.score}","{/src.flag}"]`, `so to read "{/src.score}" as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
|
||||||
{"a typed column read that holds the datatype declared beside it", `[{"format":"1.5","datatype":"number"},"{/src.score}"]`, `so write "{/src.score}" as {"format":"{/src.score}","datatype":"number"}`},
|
{"a typed column read that holds the datatype declared beside it", `[{"format":"1.5","datatype":"number"},"{/src.score}"]`, `so write "{/src.score}" as {"format":"{/src.score}","datatype":"number"}`},
|
||||||
{"text beside a weighted typed column read", `[{"format":"{/src.score}","weight":3},"n/a"]`, `to read that column as text, set its "format" to "{text}" and add "text": "{/src.score}"`},
|
{"a typed column read holding the datatype of another", `["{/src.ratio}","{/src.score}"]`, `so write "{/src.ratio}" as {"format":"{/src.ratio}","datatype":"integer"}`},
|
||||||
|
{"text before a typed column it reads", `["n/a","{/src.score}"]`, `to read that column as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
|
||||||
|
{"a weighted item beside a typed column it reads", `["{/src.score}",{"format":"5","weight":2}]`, `give it "datatype": "integer"`},
|
||||||
|
{"a typed column read before the datatype it holds", `["{/src.score}",{"format":"1.5","datatype":"number"}]`, `so write "{/src.score}" as {"format":"{/src.score}","datatype":"number"}`},
|
||||||
|
{"a typed column read beside a datatype it does not hold", `["{/src.score}",{"format":"true","datatype":"boolean"}]`, `so to read "{/src.score}" as text, write "{/src.score}" as {"format":"{/src.score}","datatype":"string"}`},
|
||||||
|
{"text beside a weighted typed column read", `[{"format":"{/src.score}","weight":3},"n/a"]`, `to read that column as text, give "{/src.score}" "datatype": "string"`},
|
||||||
{"a value of the column it reads", `{"format":"{/src.code}","datatype":"integer"}`, `"2x" is not an integer`},
|
{"a value of the column it reads", `{"format":"{/src.code}","datatype":"integer"}`, `"2x" is not an integer`},
|
||||||
{"a null read into text", `{"format":"{x}","x":"{/src.score}","datatype":"integer"}`, "reads a null"},
|
{"a null read into text", `{"format":"{x}","x":"{/src.score}","datatype":"integer"}`, "reads a null"},
|
||||||
{"a sample with leading zeros", `{"format":"{digits(3)}","datatype":"integer"}`, "{digits(3)} prints text, not an integer"},
|
{"a sample with leading zeros", `{"format":"{digits(3)}","datatype":"integer"}`, "{digits(3)} prints text, not an integer"},
|
||||||
{"a fraction", `{"format":"{v}","v":["1","1.5"],"datatype":"integer"}`, `"1.5" is not an integer`},
|
{"a fraction", `{"format":"{v}","v":["1","1.5"],"datatype":"integer"}`, `"1.5" is not an integer`},
|
||||||
{"past int64", `{"format":"9223372036854775808","datatype":"integer"}`, "past the int64 range"},
|
{"past int64", `{"format":"9223372036854775808","datatype":"integer"}`, "past the int64 range"},
|
||||||
{"a float sample", `{"format":"{float(0,1,2)}","datatype":"integer"}`, "{float(0,1,2)} prints a number, not an integer"},
|
{"a float sample", `{"format":"{float(0,1,2)}","datatype":"integer"}`, "{float(0,1,2)} prints a number, not an integer"},
|
||||||
{"composed digits", `{"format":"1{digits(2)}","datatype":"integer"}`, "is not one value"},
|
{"composed digits", `{"format":"1{digits(2)}","datatype":"integer"}`, "is not one value; write one literal or one {float()}, {int()}, {seq()} or {calc()}, or read one"},
|
||||||
{"a sign before a sample", `{"format":"-{int(1,9)}","datatype":"integer"}`, "is not one value"},
|
{"a sign before a sample", `{"format":"-{int(1,9)}","datatype":"integer"}`, "is not one value"},
|
||||||
{"a repeat", `{"format":"{int(1,9)}","repeat":2,"separator":",","datatype":"integer"}`, "carries a repeat"},
|
{"a repeat", `{"format":"{int(1,9)}","repeat":2,"separator":",","datatype":"integer"}`, "carries a repeat"},
|
||||||
{"through a reference", `{"format":"{/cat.code}","datatype":"integer"}`, `"2x" is not an integer`},
|
{"through a reference", `{"format":"{/cat.code}","datatype":"integer"}`, `"2x" is not an integer`},
|
||||||
@@ -75,7 +72,7 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
|
|||||||
{"a divisor that can be zero", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{int(0,9)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
|
{"a divisor that can be zero", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{int(0,9)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
|
||||||
{"an overflow", `{"format":"{calc(a * a)}","a":"{digits(200)}","datatype":"number"}`, "is not proven within 1e300"},
|
{"an overflow", `{"format":"{calc(a * a)}","a":"{digits(200)}","datatype":"number"}`, "is not proven within 1e300"},
|
||||||
{"a division in an integer column", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{int(1,9)}","datatype":"integer"}`, "prints a number, not an integer"},
|
{"a division in an integer column", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{int(1,9)}","datatype":"integer"}`, "prints a number, not an integer"},
|
||||||
{"a composed operand", `{"format":"{calc(n * 2)}","n":"{int(1,99)}.{digits(2)}","datatype":"number"}`, "{seq()}, {digits()} or {calc()}"},
|
{"a composed operand", `{"format":"{calc(n * 2)}","n":"{int(1,99)}.{digits(2)}","datatype":"number"}`, "one {digits()}, {float()}, {int()}, {seq()} or {calc()}"},
|
||||||
{"a divisor that prints as zero", `{"format":"{calc(1 / b, 2)}","b":"{float(4.9999999999999994e-79,5e-79,78)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
|
{"a divisor that prints as zero", `{"format":"{calc(1 / b, 2)}","b":"{float(4.9999999999999994e-79,5e-79,78)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
|
||||||
{"a divisor that rounds to zero", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{float(0.1,1,0)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
|
{"a divisor that rounds to zero", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":"{float(0.1,1,0)}","datatype":"number"}`, "divides by b, which is not proven nonzero"},
|
||||||
{"a zero among a divisor's literals", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":["0","5"],"datatype":"number"}`, "divides by b, which is not proven nonzero"},
|
{"a zero among a divisor's literals", `{"format":"{calc(a / b)}","a":"{int(1,9)}","b":["0","5"],"datatype":"number"}`, "divides by b, which is not proven nonzero"},
|
||||||
@@ -85,15 +82,13 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
|
|||||||
{"an operand past the limit", `{"format":"{calc(a)}","a":"{digits(400)}","datatype":"number"}`, "is not proven within 1e300"},
|
{"an operand past the limit", `{"format":"{calc(a)}","a":"{digits(400)}","datatype":"number"}`, "is not proven within 1e300"},
|
||||||
{"a whole calc past int64", `{"format":"{calc(a * 2)}","a":"{seq()}","datatype":"integer"}`, "{calc(a * 2)} is not proven within int64"},
|
{"a whole calc past int64", `{"format":"{calc(a * 2)}","a":"{seq()}","datatype":"integer"}`, "{calc(a * 2)} is not proven within int64"},
|
||||||
{"a whole float past int64", `{"format":"{float(0,1e19,0)}","datatype":"integer"}`, "{float(0,1e19,0)} is not proven within int64"},
|
{"a whole float past int64", `{"format":"{float(0,1e19,0)}","datatype":"integer"}`, "{float(0,1e19,0)} is not proven within int64"},
|
||||||
{"a signed zero integer", `{"format":"-0","datatype":"integer"}`, `"-0" is zero written with a sign; write "0"`},
|
|
||||||
{"a signed zero number", `{"format":"-0.00","datatype":"number"}`, `"-0.00" is zero written with a sign; write "0.00"`},
|
|
||||||
} {
|
} {
|
||||||
row := `{"format":"","col":` + c.column + `}`
|
row := `{"format":"","col":` + c.column + `}`
|
||||||
files := map[string]string{"row": row}
|
files := map[string]string{"row": row}
|
||||||
for name, body := range tree {
|
for name, body := range tree {
|
||||||
files[name] = body
|
files[name] = body
|
||||||
}
|
}
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, files)))
|
_, err := New(WithSeed(1), WithDataPath(writeData(t, files)))
|
||||||
if err == nil || !strings.Contains(err.Error(), c.want) {
|
if err == nil || !strings.Contains(err.Error(), c.want) {
|
||||||
t.Errorf("%s: New = %v, want an error containing %q", c.name, err, c.want)
|
t.Errorf("%s: New = %v, want an error containing %q", c.name, err, c.want)
|
||||||
}
|
}
|
||||||
@@ -102,7 +97,19 @@ func TestDatatypeRejectsAValueItsTypeRejects(t *testing.T) {
|
|||||||
t.Errorf("%s: NewTemplate = %v, want the inline template refused the same way", c.name, err)
|
t.Errorf("%s: NewTemplate = %v, want the inline template refused the same way", c.name, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
|
for column, want := range map[string]string{
|
||||||
|
`[{"format":"1","datatype":"integer"},{"format":"true","datatype":"boolean"}]`: "its items hold integer and boolean; a column holds one datatype",
|
||||||
|
`[{"format":"1","datatype":"integer"},{"format":"1.5","datatype":"number"}]`: "its items hold integer and number; a column holds one datatype",
|
||||||
|
} {
|
||||||
|
row := `{"format":"","col":` + column + `}`
|
||||||
|
if _, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{"row": row}))); err == nil || !strings.HasSuffix(err.Error(), want) {
|
||||||
|
t.Errorf("items of two declared datatypes %s: New = %v, want an error ending %q", column, err, want)
|
||||||
|
}
|
||||||
|
if _, err := newGenerator(t, writeData(t, tree)).NewTemplate(row); err == nil || !strings.HasSuffix(err.Error(), want) {
|
||||||
|
t.Errorf("items of two declared datatypes %s: NewTemplate = %v, want an error ending %q", column, err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, err := New(WithSeed(1), WithDataPath(writeData(t, map[string]string{
|
||||||
"a": `{"format":"","c":["{/b.x}",{"format":"1","datatype":"integer"}]}`,
|
"a": `{"format":"","c":["{/b.x}",{"format":"1","datatype":"integer"}]}`,
|
||||||
"b": `{"format":"","x":["2",{"format":"1","datatype":"integer"}]}`,
|
"b": `{"format":"","x":["2",{"format":"1","datatype":"integer"}]}`,
|
||||||
})))
|
})))
|
||||||
@@ -141,7 +148,7 @@ func TestDatatypeAcceptsAColumnThatAlwaysParses(t *testing.T) {
|
|||||||
} {
|
} {
|
||||||
row := `{"format":"","col":` + column + `}`
|
row := `{"format":"","col":` + column + `}`
|
||||||
src := `{"format":"","code":["200","404"],"n":{"format":"{int(1,9)}","datatype":"integer"}}`
|
src := `{"format":"","code":["200","404"],"n":{"format":"{int(1,9)}","datatype":"integer"}}`
|
||||||
f, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": cat, "row": row, "src": src})), WithSeed(1))
|
f, err := New(WithDataPath(writeData(t, map[string]string{"cat": cat, "row": row, "src": src})), WithSeed(1))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("%s: New = %v, want it loaded", column, err)
|
t.Errorf("%s: New = %v, want it loaded", column, err)
|
||||||
continue
|
continue
|
||||||
@@ -204,3 +211,42 @@ func TestNullColumn(t *testing.T) {
|
|||||||
t.Errorf("List() = %v, want the null column row.gone, which Fake accepts", f.List())
|
t.Errorf("List() = %v, want the null column row.gone, which Fake accepts", f.List())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDisagreementRefusesTwoTextItems(t *testing.T) {
|
||||||
|
mustPanic(t, "two text items", func() { _ = disagreement(&template{}, DataTypeString, &template{}, DataTypeString) })
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestASignedZeroIsZero(t *testing.T) {
|
||||||
|
f := newGenerator(t, writeData(t, map[string]string{"row": `{"format":"","i":{"format":"-0","datatype":"integer"},"n":{"format":"-0.00","datatype":"number"}}`}))
|
||||||
|
r, err := f.FakeRecord("row")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if c := r.Columns(); c[0].Value != "-0" || c[1].Value != "-0.00" {
|
||||||
|
t.Errorf("columns = %+v, want each written as its format", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestADeclaredDatatypeOverAColumnRead(t *testing.T) {
|
||||||
|
f := newGenerator(t, writeData(t, map[string]string{
|
||||||
|
"src": `{"format":"","score":[null,{"format":"{int(1,9)}","datatype":"integer"}]}`,
|
||||||
|
"row": `{"format":"","same":{"format":"{/src.score}","datatype":"integer"},"text":{"format":"{/src.score}","datatype":"string"}}`,
|
||||||
|
}), WithSeed(1))
|
||||||
|
empty := 0
|
||||||
|
for i := 0; i < 50; i++ {
|
||||||
|
r, err := f.FakeRecord("row")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
same, text := r.Columns()[0], r.Columns()[1]
|
||||||
|
if same.DataType != DataTypeInteger || text.DataType != DataTypeString || text.Null {
|
||||||
|
t.Fatalf("columns = %+v, want an integer column restated and a string column never null", r.Columns())
|
||||||
|
}
|
||||||
|
if text.Value == "" {
|
||||||
|
empty++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if empty == 0 {
|
||||||
|
t.Error(`50 records never read a null as "", want the string column to render one so`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
// Package fejkdata generates locale-aware fake data for tests and fixtures from JSON templates.
|
||||||
|
//
|
||||||
|
// Data lives in JSON, not in Go, and comes in modules: a Go package exporting an fs.FS,
|
||||||
|
// or a folder. A generator loads the modules you name, such as data.Modules(), every
|
||||||
|
// module fejkdata ships, from github.com/larvit/fejkdata/data. Their folders and files
|
||||||
|
// become a dot-path namespace, and you generate values by path:
|
||||||
|
//
|
||||||
|
// f, _ := fejkdata.New(fejkdata.WithDataFS(data.Modules()...), fejkdata.WithSeed(42))
|
||||||
|
// f.Fake("sv_SE.address") // "Trekungagatan 43\n442 42 Kungälv"
|
||||||
|
// f.Fake("sv_SE.address.locality") // "Umeå": a second call draws afresh
|
||||||
|
//
|
||||||
|
// Several sources merge in order, the last winning a name clash, so custom data
|
||||||
|
// layers over the built-ins. The JSON template format is documented in the [README].
|
||||||
|
//
|
||||||
|
// [README]: https://github.com/larvit/fejkdata#readme
|
||||||
|
package fejkdata
|
||||||
|
|
||||||
|
// Vocabulary
|
||||||
|
//
|
||||||
|
// - draw — one item taken from a choice, or one row from a table: `drawItem`,
|
||||||
|
// `drawThroughChoices`, `renderEnv.drawRowOf`. Every {…} draws afresh, bar a read of a name.
|
||||||
|
// - expansion — one render of one format: `expand`.
|
||||||
|
// - render — one walk of the tree from an entry point, over one `frameStack`:
|
||||||
|
// `Generator.Fake`, `FakeRecord`, `Template.Fake` and `RecordTemplate.Fake`
|
||||||
|
// start one each, and `FakeStruct` one per record.
|
||||||
|
// - env — what a render reads names and rows through: `renderEnv`.
|
||||||
|
// - scope — where a name is bound: a category, or one repeat iteration: `nameScope`.
|
||||||
|
// - named pick — the draw a token such as {/person as p} binds to a name, drawn
|
||||||
|
// on its first read and kept while its scope renders: `nameBinding`,
|
||||||
|
// `namedPick`, `pickFrame`.
|
||||||
|
// - level, pick key, addressed key — `pickKey`.
|
||||||
|
// - memo — what a named pick keeps its draws in, the variant drawn at each level
|
||||||
|
// and the value each path read: `drawMemo`.
|
||||||
|
// - pin — fixing which row of a table one path, or one named pick, uses:
|
||||||
|
// `pinSet`, `namedPick.pins`, `table.followStep`.
|
||||||
|
// - link — a table's tie to its parent table, each row to its parent row:
|
||||||
|
// `table.linkParent`.
|
||||||
|
// - family — a table and every table reaching it through a chain of parent
|
||||||
|
// links.
|
||||||
|
// - resolve — tying a reference or a name to what it names in the assembled tree:
|
||||||
|
// `resolveTemplates`, `resolveRefs`.
|
||||||
|
// - head — the string a read starts from: a field or a name as written, or a
|
||||||
|
// reference's category path with its sigil, /sv_SE.address: `arm.head`. The node it names is
|
||||||
|
// the read's start: `template.startOf`, `nameTarget.start`.
|
||||||
|
// - fence — a load-time check, run over each category, template or struct as it
|
||||||
|
// loads or compiles, so rendering a compiled tree cannot fail.
|
||||||
+286
@@ -0,0 +1,286 @@
|
|||||||
|
package fejkdata
|
||||||
|
|
||||||
|
import (
|
||||||
|
"go/ast"
|
||||||
|
"go/build"
|
||||||
|
"go/parser"
|
||||||
|
"go/token"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Every backtick in the Vocabulary is a symbol.
|
||||||
|
var backticked = regexp.MustCompile("`([^`\n]+)`")
|
||||||
|
|
||||||
|
// TestVocabularyNamesDeclaredSymbols holds a renamed unit to its Vocabulary entry.
|
||||||
|
func TestVocabularyNamesDeclaredSymbols(t *testing.T) {
|
||||||
|
_, files := sourceFiles(t)
|
||||||
|
declared := declaredSymbols(files)
|
||||||
|
if len(namesDeclared(t, "the Vocabulary", vocabulary(t, files), declared)) == 0 {
|
||||||
|
t.Fatal("the Vocabulary names no symbol, so nothing holds a renamed unit to its entry")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// namesDeclared is every symbol text backticks, failing for each one the package
|
||||||
|
// declares nowhere.
|
||||||
|
func namesDeclared(t *testing.T, where, text string, declared map[string]bool) []string {
|
||||||
|
t.Helper()
|
||||||
|
var named []string
|
||||||
|
for _, m := range backticked.FindAllStringSubmatch(text, -1) {
|
||||||
|
if !declared[m[1]] {
|
||||||
|
t.Errorf("%s names %s, which the package declares nowhere", where, m[1])
|
||||||
|
}
|
||||||
|
named = append(named, m[1])
|
||||||
|
}
|
||||||
|
return named
|
||||||
|
}
|
||||||
|
|
||||||
|
// namesAPass fails unless one symbol text backticks is a function: a field names
|
||||||
|
// what a pass reads or writes, never the pass.
|
||||||
|
func namesAPass(t *testing.T, where, text string, declared, funcs map[string]bool) {
|
||||||
|
t.Helper()
|
||||||
|
named := namesDeclared(t, where, text, declared)
|
||||||
|
if !slices.ContainsFunc(named, func(n string) bool { return funcs[n] }) {
|
||||||
|
t.Errorf("%s names no function, so it names no pass", where)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func packageDir(t *testing.T) *build.Package {
|
||||||
|
t.Helper()
|
||||||
|
pkg, err := build.ImportDir(".", 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read the package directory: %v", err)
|
||||||
|
}
|
||||||
|
return pkg
|
||||||
|
}
|
||||||
|
|
||||||
|
func sourceFiles(t *testing.T) (*token.FileSet, []*ast.File) {
|
||||||
|
t.Helper()
|
||||||
|
return parseFiles(t, packageDir(t).GoFiles)
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseFiles(t *testing.T, names []string) (*token.FileSet, []*ast.File) {
|
||||||
|
t.Helper()
|
||||||
|
fset := token.NewFileSet()
|
||||||
|
var files []*ast.File
|
||||||
|
for _, name := range names {
|
||||||
|
f, err := parser.ParseFile(fset, name, nil, parser.ParseComments)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse %s: %v", name, err)
|
||||||
|
}
|
||||||
|
files = append(files, f)
|
||||||
|
}
|
||||||
|
return fset, files
|
||||||
|
}
|
||||||
|
|
||||||
|
func vocabulary(t *testing.T, files []*ast.File) string {
|
||||||
|
t.Helper()
|
||||||
|
var sections strings.Builder
|
||||||
|
for _, f := range files {
|
||||||
|
for _, c := range f.Comments {
|
||||||
|
section, found := strings.CutPrefix(c.Text(), "Vocabulary\n\n")
|
||||||
|
if found && c.Pos() > f.Package {
|
||||||
|
sections.WriteString(section)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if sections.Len() == 0 {
|
||||||
|
t.Fatal("no comment below a package clause opens with a Vocabulary heading, so the words the package is written in are defined nowhere a reader of the code meets them")
|
||||||
|
}
|
||||||
|
return sections.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// declaredSymbols is every name the package declares at the top level, a method and
|
||||||
|
// a struct field keyed under its type: namedPick.pins, tableRead.landsRow.
|
||||||
|
func declaredSymbols(files []*ast.File) map[string]bool {
|
||||||
|
names := declaredFuncs(files)
|
||||||
|
for _, f := range files {
|
||||||
|
for _, decl := range f.Decls {
|
||||||
|
if d, isGen := decl.(*ast.GenDecl); isGen {
|
||||||
|
addSpecs(names, d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
// declaredFuncs is every function the package declares, a method keyed under its
|
||||||
|
// type: template.fixedText.
|
||||||
|
func declaredFuncs(files []*ast.File) map[string]bool {
|
||||||
|
names := map[string]bool{}
|
||||||
|
for _, f := range files {
|
||||||
|
for _, decl := range f.Decls {
|
||||||
|
d, isFunc := decl.(*ast.FuncDecl)
|
||||||
|
if !isFunc {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
names[d.Name.Name] = true
|
||||||
|
if recv := receiverType(d.Recv); recv != "" {
|
||||||
|
names[recv+"."+d.Name.Name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
func addSpecs(names map[string]bool, d *ast.GenDecl) {
|
||||||
|
for _, spec := range d.Specs {
|
||||||
|
switch s := spec.(type) {
|
||||||
|
case *ast.TypeSpec:
|
||||||
|
names[s.Name.Name] = true
|
||||||
|
addFields(names, s)
|
||||||
|
case *ast.ValueSpec:
|
||||||
|
for _, id := range s.Names {
|
||||||
|
names[id.Name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func addFields(names map[string]bool, s *ast.TypeSpec) {
|
||||||
|
st, isStruct := s.Type.(*ast.StructType)
|
||||||
|
if !isStruct {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, field := range st.Fields.List {
|
||||||
|
for _, id := range field.Names {
|
||||||
|
names[s.Name.Name+"."+id.Name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func receiverType(recv *ast.FieldList) string {
|
||||||
|
if recv == nil || len(recv.List) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
e := recv.List[0].Type
|
||||||
|
if star, isPointer := e.(*ast.StarExpr); isPointer {
|
||||||
|
e = star.X
|
||||||
|
}
|
||||||
|
id, isIdent := e.(*ast.Ident)
|
||||||
|
if !isIdent {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return id.Name
|
||||||
|
}
|
||||||
|
|
||||||
|
const fillHeader = "Filled by "
|
||||||
|
|
||||||
|
// TestTemplateFieldGroupsNameADeclaredPass holds a renamed pass to the group it
|
||||||
|
// fills, and to the template doc.
|
||||||
|
func TestTemplateFieldGroupsNameADeclaredPass(t *testing.T) {
|
||||||
|
fset, files := sourceFiles(t)
|
||||||
|
declared, funcs := declaredSymbols(files), declaredFuncs(files)
|
||||||
|
doc, st, headers := templateStruct(t, fset, files)
|
||||||
|
namesAPass(t, "the template doc", doc, declared, funcs)
|
||||||
|
for _, field := range st.Fields.List {
|
||||||
|
if len(field.Names) == 0 {
|
||||||
|
t.Error("an embedded field of template sits under no header, so nothing says which pass fills it")
|
||||||
|
}
|
||||||
|
for _, id := range field.Names {
|
||||||
|
header := headerAbove(headers, id.Pos())
|
||||||
|
if header == "" {
|
||||||
|
t.Errorf("template.%s sits under no %q header, so nothing says which pass fills it", id.Name, fillHeader)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
namesAPass(t, "the header above template."+id.Name, header, declared, funcs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// templateStruct is the template struct, its doc, and the fill headers standing
|
||||||
|
// between its fields, in source order.
|
||||||
|
func templateStruct(t *testing.T, fset *token.FileSet, files []*ast.File) (string, *ast.StructType, []*ast.CommentGroup) {
|
||||||
|
t.Helper()
|
||||||
|
for _, f := range files {
|
||||||
|
for _, decl := range f.Decls {
|
||||||
|
d, isGen := decl.(*ast.GenDecl)
|
||||||
|
if !isGen {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, spec := range d.Specs {
|
||||||
|
s, isType := spec.(*ast.TypeSpec)
|
||||||
|
if !isType || s.Name.Name != "template" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
st, isStruct := s.Type.(*ast.StructType)
|
||||||
|
if !isStruct {
|
||||||
|
t.Fatal("template is not a struct")
|
||||||
|
}
|
||||||
|
return d.Doc.Text() + s.Doc.Text(), st, fillHeaders(fset, f, st)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatal("the package declares no template struct")
|
||||||
|
return "", nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// fillHeaders skips a header trailing a field: it names that field, never the
|
||||||
|
// fields below it.
|
||||||
|
func fillHeaders(fset *token.FileSet, f *ast.File, st *ast.StructType) []*ast.CommentGroup {
|
||||||
|
trailing := map[int]bool{}
|
||||||
|
for _, field := range st.Fields.List {
|
||||||
|
trailing[fset.Position(field.End()).Line] = true
|
||||||
|
}
|
||||||
|
var found []*ast.CommentGroup
|
||||||
|
for _, c := range f.Comments {
|
||||||
|
if c.Pos() < st.Pos() || c.End() > st.End() || trailing[fset.Position(c.Pos()).Line] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(c.Text(), fillHeader) {
|
||||||
|
found = append(found, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return found
|
||||||
|
}
|
||||||
|
|
||||||
|
// headerAbove is the last fill header standing above pos.
|
||||||
|
func headerAbove(headers []*ast.CommentGroup, pos token.Pos) string {
|
||||||
|
text := ""
|
||||||
|
for _, c := range headers {
|
||||||
|
if c.Pos() < pos {
|
||||||
|
text = c.Text()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return text
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
printedCall = regexp.MustCompile(`(?m)^.*Fake\w*\(.*//\s*".*$`)
|
||||||
|
printedFake = regexp.MustCompile(`\.Fake\("([^"]+)"\)\s*//\s*("(?:[^"\\]|\\.)*")`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestGoExamplesPrintWhatTheyRender replays each Go example's printed Fake calls on seed 42.
|
||||||
|
func TestGoExamplesPrintWhatTheyRender(t *testing.T) {
|
||||||
|
for _, name := range []string{"doc.go", "README.md"} {
|
||||||
|
src, err := os.ReadFile(name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
lines := printedCall.FindAllString(string(src), -1)
|
||||||
|
if lines == nil {
|
||||||
|
t.Errorf("%s prints no seeded Fake, so nothing holds its example to a render", name)
|
||||||
|
}
|
||||||
|
f, err := New(withShipped(), WithSeed(42))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, line := range lines {
|
||||||
|
c := printedFake.FindStringSubmatch(line)
|
||||||
|
if c == nil {
|
||||||
|
t.Fatalf("%s prints a value this test cannot replay; spell it f.Fake(\"path\") // \"value\": %s", name, line)
|
||||||
|
}
|
||||||
|
want, err := strconv.Unquote(c[2])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s prints %s: %v", name, c[2], err)
|
||||||
|
}
|
||||||
|
if got := fake(t, f, c[1]); got != want {
|
||||||
|
t.Errorf("%s prints Fake(%q) = %q for seed 42, it renders %q", name, c[1], want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,722 @@
|
|||||||
|
# Comprehension history
|
||||||
|
|
||||||
|
## 2026-10-09T08:59:19Z, PR #190 at 17f732a, against ee4b40b
|
||||||
|
|
||||||
|
Ruling: better
|
||||||
|
|
||||||
|
| Seat | Vote |
|
||||||
|
|---|---|
|
||||||
|
| Mid A | better |
|
||||||
|
| Inherited architect | better |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- `pick.go:9` `pickKey`: the definition sits on the type, beside `levelKeys`, `addressedKeys`, `keptInPick` and `readUnder`, where four files held them
|
||||||
|
- `render.go:155` `readField`: one named predicate, `keptInPick`, states the whole keep rule
|
||||||
|
- `datatype.go:217` `columnRead`: beside the code that reads it
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- `pick.go:9` `pickKey`: the 3am path for a named pick now ends in one file, with its worked example
|
||||||
|
- `render.go:155` `readField`: `keptInPick` holds its own nil and reference checks
|
||||||
|
- `path.go:72` `compiledPath`: the field no longer reuses the vocabulary word level
|
||||||
|
|
||||||
|
## 2026-10-09T09:24:31Z, PR #190 at 65a32db, against ee4b40b
|
||||||
|
|
||||||
|
Ruling: better
|
||||||
|
|
||||||
|
| Seat | Vote |
|
||||||
|
|---|---|
|
||||||
|
| Mid A | better |
|
||||||
|
| Inherited architect | better |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- `pick.go:9` `pickKey`: the definitions sit on the type, beside the functions that build and read the keys
|
||||||
|
- `datatype.go:217` `columnRead`: beside `itemDatatype` and `checkColumns`, which consume it
|
||||||
|
- `node.go:70` `template.readsColumn`: names the four readers that change together
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- `pick.go:9` `pickKey`: a pick-key bug needs one file, where it needed four and the vocabulary
|
||||||
|
- `render.go:155` `readField`: one named predicate, `keptInPick`, states the keep rule
|
||||||
|
- `path.go:72` `compiledPath`: `headSpans` is the costliest new read in the diff
|
||||||
|
|
||||||
|
## 2026-10-09T09:30:15Z, PR #190 at d9f0fc9, against ee4b40b
|
||||||
|
|
||||||
|
Ruling: better
|
||||||
|
|
||||||
|
| Seat | Vote |
|
||||||
|
|---|---|
|
||||||
|
| Mid A | better |
|
||||||
|
| Inherited architect | better |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- `pick.go:9` `pickKey`: the definitions and the code building and reading the keys share one file
|
||||||
|
- `render.go:155` `readField`: `keptInPick` holds the whole keep rule and its own nil check
|
||||||
|
- `path.go:74` `compiledPath`: `headSpans` and the bare 0 its callers pass cost a second read
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- `pick.go:9` `pickKey`: the 3am mechanism reads in one file, in reading order
|
||||||
|
- `datatype.go:217` `columnRead`: beside its consumers, and `node.go:70` names its four readers
|
||||||
|
- `path.go:119` `compileStep`: six parameters, the one place that reads harder
|
||||||
|
|
||||||
|
## 2026-10-09T10:16:38Z, PR #193 at c27e8b7, against d74c0ed
|
||||||
|
|
||||||
|
Ruling: better
|
||||||
|
|
||||||
|
| Seat | Vote |
|
||||||
|
|---|---|
|
||||||
|
| Mid A | better |
|
||||||
|
| Inherited architect | better |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- `data-import/geo.py:75` `fence`: one named step states the locality row and drops childless parents level by level, where `with_streets` pruned localities alone
|
||||||
|
- `data-import/geo-us.py:174` `main`: the county and region pruning no longer hides inside the `tsv.write` comprehensions
|
||||||
|
- `data-import/geo-se.py:186` `main`: builds its rows before the fence, so it has the US script's shape
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- `data-import/geo.py:75` `fence`: the loader's parent-has-a-child rule is named in the import code and applied in one place at all three levels
|
||||||
|
- `data-import/geo-us.py:174` `main`: one `fence` call before any write replaces filters buried in the writes
|
||||||
|
- `data-import/geo-se.py:186` `main`: both scripts read alike, build rows, fence, write
|
||||||
|
|
||||||
|
## 2026-10-09T10:29:19Z, PR #193 at 8433689, against d74c0ed
|
||||||
|
|
||||||
|
Ruling: better
|
||||||
|
|
||||||
|
| Seat | Vote |
|
||||||
|
|---|---|
|
||||||
|
| Mid A | better |
|
||||||
|
| Inherited architect | better |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- `data-import/geo.py:75` `fence`: the parent-has-a-child rule lives in one function whose docstring states the rule, each argument's shape and what it returns
|
||||||
|
- `data-import/geo-us.py:174` `main`: `zctas` is built once and fenced, and every write reads from `kept` with no filter of its own
|
||||||
|
- `data-import/geo-us.py:128` `zcta_localities`: the name says it returns a ZCTA-to-place map, where `postal_codes` pointed at the wrong output
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- `data-import/geo.py:75` `fence`: one unit owns the chain from locality to municipality to region, where the base spread it over `with_streets` and both `main`s
|
||||||
|
- `data-import/geo-us.py:128` `zcta_localities`: the name and docstring say what it maps to
|
||||||
|
- `data-import/geo-se.py:181` `main`: `region_names` and `municipality_names` say they map codes to names, and the rows are built once before the fence
|
||||||
|
|
||||||
|
## 2026-10-07, scoring run at ad59967
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mean | 7.00 | 5.88 | 6.50 | 6.13 | 6.38 |
|
||||||
|
|
||||||
|
## 2026-10-09T11:04:29Z, PR #195 at b9d1af6
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Junior A | 7.5 | 6.5 | 7 | 7 | 7 |
|
||||||
|
| Mid A | 7.5 | 6 | 6.5 | 7 | 6.5 |
|
||||||
|
| Maintainability senior | 7.5 | 6 | 6.5 | 6.5 | 6.5 |
|
||||||
|
| Inherited architect | 7 | 5.5 | 6 | 7 | 6.5 |
|
||||||
|
| Mean | 7.38 | 6.00 | 6.50 | 6.88 | 6.63 |
|
||||||
|
|
||||||
|
Junior A, hardest first:
|
||||||
|
|
||||||
|
- `pick.go:70` `keptInPick`: a level, a first-level key, that key moved under `env.pickAt`, and the name's addressed set, held at once
|
||||||
|
- `namefence.go:76` `refuseTwiceDrawnIn`: mirrors the keep rule in `keptInPick`, and nothing ties the two in code
|
||||||
|
|
||||||
|
Mid A, hardest first:
|
||||||
|
|
||||||
|
- `pick.go:79` `readUnder`: a key built at load in `addressedKeys` must equal one built at render through `pickKey.under`, and a mismatch is silent
|
||||||
|
- `node.go:70` `readsColumn`: four functions in three files must change together
|
||||||
|
|
||||||
|
Maintainability senior, hardest first:
|
||||||
|
|
||||||
|
- `render.go:146` `readField`: whether a read is kept in a pick rests on string keys from three places and on `env.pick` cleared at the right moment
|
||||||
|
- `path.go:190` `drawSteps`: a nil `pins` or `memo` names the mode, and `climbed` carries state across iterations
|
||||||
|
|
||||||
|
Inherited architect, would restructure first:
|
||||||
|
|
||||||
|
- `pick.go:179` `readName`: keeping one named pick needs `name.go`, `pick.go`, `env.go`, `render.go`, `path.go` and `record.go` held at once
|
||||||
|
- `node.go:52` `template`: filled in four phases by four functions
|
||||||
|
|
||||||
|
## 2026-10-09T11:09:34Z, PR #195 at 344023d, against 1c342bc
|
||||||
|
|
||||||
|
Ruling: better
|
||||||
|
|
||||||
|
| Seat | Vote |
|
||||||
|
|---|---|
|
||||||
|
| Mid A | better |
|
||||||
|
| Inherited architect | better |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- `shipped.go:61` `loadCallerPath`: `Fake` and `FakeRecord` get their segments from the one call that loads them, where each repeated the split and the load
|
||||||
|
- `node.go:30` `choice`: the `shared` comment says what the field counts, where it claimed `List` reads it
|
||||||
|
- `data-import/tsv.py:7` `write`: its check and message match the loader's one-row rule
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- `shipped.go:61` `loadCallerPath`: the load-before-walk order is held by data flow, where a comment on `loadShippedAt` stated it
|
||||||
|
- `node.go:30` `choice`: the `shared` comment no longer sends a `--list` bug to the wrong field
|
||||||
|
- `README.md:163` `## Data`: the opening names the `geo` trees the section documents
|
||||||
|
|
||||||
|
## 2026-10-09T16:48:45Z, PR #201 at c3ef728, against 2652779
|
||||||
|
|
||||||
|
Ruling: same
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | same | same | same | same | same |
|
||||||
|
| Inherited architect | same | same | same | same | same |
|
||||||
|
|
||||||
|
## 2026-10-09T16:54:01Z, PR #201 at 8b5a972, against 2652779
|
||||||
|
|
||||||
|
Ruling: same
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | same | same | same | same | same |
|
||||||
|
| Inherited architect | same | same | same | same | same |
|
||||||
|
|
||||||
|
## 2026-10-09T17:21:57Z, PR #202 at 65b2de8, against 2652779
|
||||||
|
|
||||||
|
Ruling: worse
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | same | worse | same | same | worse |
|
||||||
|
| Inherited architect | same | worse | worse | worse | worse |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `data.go:13` `loadSources`: one pass mixes lazy and eager loading through before/after snapshots, a `provides` closure and a reverse graph search, and the skip of the first source is unexplained
|
||||||
|
- Locality: `data.go:107` `mergeFolder`: a category must sit in one of two maps, `children` or `unloaded`, kept by deletes in four functions across two files
|
||||||
|
- Locality: `internal/datafiles/datafiles.go:83` `embeddedManifests`: a process-wide cache in a package that was otherwise pure
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- Locality, Self-sufficiency: `data.go:13` `loadSources`: nothing says why replaced paths drag their readers in, or what the `!after[p]` case covers
|
||||||
|
- Locality: `index.go:113` `loadReached`: its rollback claims the next call fails the same way, which holds only while no parent table was linked to a rolled-back child
|
||||||
|
- Shape: `index.go:159` `reaching`: four near-synonyms, `reaching`, `reachedFrom`, `loadReached` and `referenced`, and `referenced` and `categoryUnder` return loaded categories too
|
||||||
|
- Shape: `data.go:40` `place`: shares its name with `compileInto`'s parameter, and returns a closure answering one question two ways
|
||||||
|
- Locality: `internal/datafiles/datafiles.go:83` `embeddedManifests`: hidden global state for a small saving
|
||||||
|
|
||||||
|
## 2026-10-09T17:32:44Z, PR #202 at fdc6a04, against 2652779
|
||||||
|
|
||||||
|
Ruling: worse
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | same | worse | same | same | worse |
|
||||||
|
| Inherited architect | same | worse | worse | worse | worse |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `index.go:175` `loadReached`: one loop loads, puts back on failure, checks each entry before linking and again after the pipeline, and its rollback rests on an ordering invariant its comment argues
|
||||||
|
- Locality: `data.go:13` `loadSources`: whether a category loads in New means holding every source and every index entry together, through `standing` snapshots and `dependents`
|
||||||
|
- Shape: `index.go:36` `categoryAt`: a near-synonym of `categorySite`, beside `dependencies` and `dependents`, which differ by two letters and run in opposite directions
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `index.go:175` `loadReached`: five jobs in one queue loop, with a cross-call ordering invariant to hold
|
||||||
|
- Locality, Shape: `data.go:13` `loadSources`: one source can make another load early through identity-compared snapshots and a reverse graph over index entries
|
||||||
|
- Shape: `index.go:36` `categoryAt`: two near-synonym types for where a category sits, and three near-identical function names
|
||||||
|
- Self-sufficiency: `index.go:248` `staleEntry`: its note on linking cannot be read without knowing the pipeline, and `config.load` points at a paragraph-long decision anchor
|
||||||
|
|
||||||
|
## 2026-10-09T17:39:40Z, PR #202 at 0bdb100, against 2652779
|
||||||
|
|
||||||
|
Ruling: worse
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | same | worse | worse | same | worse |
|
||||||
|
| Inherited architect | same | worse | worse | same | worse |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `data.go:13` `loadSources`: whether an indexed category loads in New depends on source order and what other sources hold, through before-and-after snapshots
|
||||||
|
- Locality: `index.go:119` `affectedBy`: trusts the manifest's `reads` to match the templates, with nothing beside the code checking it
|
||||||
|
- Locality, Shape: `index.go:174` `loadReached`: a rollback batch to reason about, with names that say less than `unloadedReads` and `linkedTables` did
|
||||||
|
- Shape: `index.go:16` `ErrLoad`: a public error every entry point depends on, in a file named for the index
|
||||||
|
- Shape: `internal/datafiles/datafiles.go:103` `decodeManifest`: a hand-written strict decoder doubles the package's responsibilities
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `data.go:13` `loadSources`: New's load set follows dependencies forward from templates and backward from the manifest's `reads`, and a stale `reads` leaves a broken reader for its first reach
|
||||||
|
- Locality: `index.go:201` `batch.load`: the unloaded entry is deleted three calls away, and `putBack` reverses it
|
||||||
|
- Locality, Shape: `data.go:62` `standing`: a `map[string]any` of nodes and entries compared by identity
|
||||||
|
- Shape: `index.go:1` `index.go`: fifteen units under generic names in a file named for the index, `ErrLoad` among them
|
||||||
|
- Shape: `internal/datafiles/datafiles.go:103` `decodeManifest`: a strict JSON decoder shares the tree walker's file with no seam
|
||||||
|
|
||||||
|
## 2026-10-10T09:09:41Z, PR #202 at d34cc42, against 2652779
|
||||||
|
|
||||||
|
Ruling: better
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | same | better | better | same | better |
|
||||||
|
| Inherited architect | same | better | same | same | better |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `data.go:12` `loadSources`: one path, each source indexed or walked then one `loadReached`, where `config.load` switched between a lazy and an eager mode
|
||||||
|
- Locality: `index.go:107` `loadReached`: a failed load puts back what it loaded and returns an error, where `loadShipped` panicked partway through
|
||||||
|
- Locality, Shape: `node.go:31` `folder.put`: two methods keep a name in `children` or `unloaded`, never both
|
||||||
|
- Shape: `index.go:29` `categoryAt`: `shipped.go` is only the embed, and `categorySite` embeds `categoryAt` in place of a duplicated `unloadedCategory`
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- Locality: `data.go:12` `loadSources`: one load path, where `config.load` hid a mode switch between lazy and eager loading
|
||||||
|
- Locality: `index.go:107` `loadReached`: returns a load error, and names a manifest entry drifted from its table, where `loadShipped` panicked on a hand-regenerated Go file
|
||||||
|
- Locality: `node.go:23` `folder`: the children-or-unloaded rule stated once and kept by two methods
|
||||||
|
|
||||||
|
## 2026-10-10T09:21:22Z, PR #202 at b77ec94, against 2652779
|
||||||
|
|
||||||
|
Ruling: better
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | same | better | same | same | better |
|
||||||
|
| Inherited architect | same | worse | same | same | worse |
|
||||||
|
| Maintainability senior | same | better | same | better | better |
|
||||||
|
| Junior A | same | better | better | same | better |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `node.go:31` `folder.put`: two methods keep a name in `children` or `unloaded`, where `unloadedCategory.load` kept it by a separate delete
|
||||||
|
- Locality, Overall: `data.go:22` `addSource`: each source's own manifest decides how it loads, where adding any data path switched the whole shipped set to eager loading
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `index.go:109` `loadReached`: one function serves New and first reach, rolls back through `putBack`, and rests on facts held elsewhere
|
||||||
|
- Locality: `index.go:22` `placeIndex`: how sources merge depends on their order and on which kind each is, and the children-or-unloaded rule holds only while `put` is the one writer
|
||||||
|
|
||||||
|
Maintainability senior, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `data.go:12` `loadSources`: every source goes through one path, where `config.load` branched between two load paths of different eagerness
|
||||||
|
- Locality: `index.go:15` `indexed`: an unloaded entry carries its own source, where `unloadedCategory.load` was wired to the global `shippedSource`
|
||||||
|
- Self-sufficiency, Overall: `internal/datafiles/manifest.go:56` `decodeManifest`: the index is data checked strictly, where `shippedEntry` told the reader to empty a generated literal by hand
|
||||||
|
|
||||||
|
Junior A, decided by:
|
||||||
|
|
||||||
|
- Locality, Shape: `node.go:23` `folder`: `put` and `putUnloaded` keep a name in one map, where `compileInto` and `unloadedCategory.load` kept it as a pair
|
||||||
|
- Locality, Shape: `data.go:12` `loadSources`: one load path, where `config.load` forked and `loadShipped` was a second, panic-only pipeline
|
||||||
|
- Shape: `data.go:81` `categorySite`: embeds `categoryAt`, where `unloadedCategory` repeated its fields
|
||||||
|
|
||||||
|
## 2026-10-10T11:11:16Z, PR #203 at 421bb49, against bf1becb
|
||||||
|
|
||||||
|
Ruling: worse
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | same | worse | worse | same | worse |
|
||||||
|
| Inherited architect | same | worse | worse | same | worse |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `node.go:28` `folder.reads`: a third per-name map beside children and unloaded with no invariant tying it to them; every placement site must remember a separate setReads call (`data.go:36` addSource, `data.go:76` mergeFolder, `index.go:27` placeIndex)
|
||||||
|
- Locality, Shape: `resolve.go:96` `withReads`: the manifest's reads travels through folder, categorySite.reads and templateSite.category to feed one error clause, four files to hold for one sentence of error text
|
||||||
|
- Shape: `internal/datafiles/manifest.go:22` `Manifest.Reads`: reads as a load dependency the loader acts on, though it only feeds error text, and "reads" already means a template reading a name or column
|
||||||
|
- Shape: `data/geo/SE/fs.go:2` `SE`: package names SE, US, en_US, sv_SE; in Modules SE.FS sits beside sv_SE.FS with nothing saying which is geography
|
||||||
|
- Locality: `fejkdata.go:94` `New`: loads exactly what the options name, easier than the hidden config{shipped: true} default, though config.given brings back a smaller flag
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- Locality, Shape, Overall: `node.go:27` `folder.reads`: a third per-name map that put and putUnloaded leave alone, so addSource, mergeFolder and placeIndex must each call setReads, and batch.load relies on placeIndex having done it; action at a distance for one error suffix
|
||||||
|
- Locality: `resolve.go:94` `templateSite.withReads`: reads travels folder, categorySite.reads, a *categorySite per template, then withReads, four hops across three files
|
||||||
|
- Shape: `data/geo/SE/fs.go:2` `SE`: bare package names SE and US do not say geography at WithDataFS(sv_SE.FS, SE.FS); the namespace-doubling rule lives only in README and test code
|
||||||
|
- Shape, Self-sufficiency: `internal/datafiles/manifest.go:22` `Manifest.Reads`: suggests it drives loading though it only feeds withReads; "by default" needs README goal 9.2
|
||||||
|
- Locality, Navigation: `data/modules.go:17` `Modules`: the hidden shippedFS default is gone; Modules lists the modules, errNoData names the fix, and the CLI loads data.Modules() in plain sight
|
||||||
|
- Locality: `fejkdata.go:40` `config.given`: WithDataFS() with no arguments differs from no option, learnt only from its comment
|
||||||
|
|
||||||
|
## 2026-10-10T11:24:33Z, PR #203 at 70e552c, against bf1becb
|
||||||
|
|
||||||
|
Ruling: same
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | same | better | worse | same | same |
|
||||||
|
| Inherited architect | worse | worse | better | same | worse |
|
||||||
|
| Maintainability senior | worse | same | worse | same | worse |
|
||||||
|
| Junior A | same | worse | same | same | same |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- Locality: `fejkdata.go:94` `New`: loads only the modules its options name or fails with errNoData; the base silently added the embedded shippedSource in config.load
|
||||||
|
- Shape: `internal/datafiles/manifest.go:23` `Manifest.Reads`: Reads and categorySite.defaultModules suggest a loaded dependency, though only a hint reads them, and "by default" is never set against anything
|
||||||
|
- Locality, Shape: `data.go:12` `loadSources`: default modules reach a category by two routes, the walked map and indexed.defaultModules, and templateSite gains a pointer only to carry them
|
||||||
|
- Shape: `data/modules.go:18` `Modules`: package names SE, US, sv_SE, en_US are unidiomatic, and every embedded path repeats itself
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `data.go:12` `loadSources`: addSource fills a side map keyed by path that loadSources copies onto sites after the merge, so merge order and clashes must be reasoned about on the core load path
|
||||||
|
- Locality, Shape: `data.go:92` `categorySite.defaultModules`: the manifest's module list passes four carriers, set at two distant places and not by siteIn, so a future caller resolving siteIn sites gets a false "names no module" message
|
||||||
|
- Navigation: `data/geo/SE/geo/SE/address.json` `geo.SE.address`: every shipped file sits under a stuttered path
|
||||||
|
- Shape: `fejkdata.go:65` `WithDataFS`: the root no longer embeds data, datafiles.Source drops base, and data sits behind data/modules.go and five small packages
|
||||||
|
- Navigation, Shape: `internal/datafiles/manifest.go:23` `Manifest.Reads`: import paths that look resolved though nothing resolves them, generated by a test reached only through go:generate
|
||||||
|
|
||||||
|
Maintainability senior, decided by:
|
||||||
|
|
||||||
|
- Navigation, Overall: `data/geo/SE/geo/SE/address.json` `geo.SE.address`: the path repeats the namespace inside the module, one extra hop from category to file, two for geo
|
||||||
|
- Shape: `internal/datafiles/manifest.go:21` `Manifest.Reads`: reads sounds like a loaded dependency, and the same list is also defaultModules and walked
|
||||||
|
- Navigation, Shape: `data/modules.go:15` `go:generate`: regenerating a module's manifest runs a test one package up
|
||||||
|
- Shape: `data.go:20` `loadSources`: defaultModules reaches categorySite by two routes, and siteIn leaves it empty
|
||||||
|
|
||||||
|
Junior A, decided by:
|
||||||
|
|
||||||
|
- Locality: `data.go:12` `loadSources`: default modules pass through four files, an out-parameter map patched onto sites after the merge for walked sources and indexed.defaultModules copied in batch.load for indexed ones
|
||||||
|
- Locality: `fejkdata.go:94` `New`: reads straight through with no hidden default source, which only partly offsets the above
|
||||||
|
|
||||||
|
## 2026-10-11T09:52:08Z, PR #203 at 4c4bbcc, against bf1becb
|
||||||
|
|
||||||
|
Ruling: worse
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | same | same | worse | same | worse |
|
||||||
|
| Inherited architect | worse | same | worse | same | worse |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- Shape, Overall: `data/geo/SE/fs.go:2` `SE`: each module embeds a tree repeating its own name, and the package names SE, US, en_US, sv_SE are unidiomatic; five thin packages plus a list replace one embed
|
||||||
|
- Shape: `reference.go:124` `withDefaultModules`: "default modules" suggests modules that load by default, though nothing loads them; "by default" is spelled out only in the README
|
||||||
|
- Locality, Shape: `inline.go:40` `NewTemplate`: errors are decorated twice, by hand, at four entry points
|
||||||
|
- Locality: `fejkdata.go:101` `New`: the hidden shipped default is gone, offsetting the above
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- Navigation, Overall: `data/sv_SE/fs.go:8` `FS`: every data file sits under a doubled folder, data/sv_SE/sv_SE/, data/geo/SE/geo/SE/, a stop at each hop of the 3am path
|
||||||
|
- Navigation: `data/modules.go:15` `go:generate`: the module list is written twice and the manifest writer lives in a root test run from the data package
|
||||||
|
- Shape: `data/geo/SE/fs.go:2` `SE`: a package named SE reads as Sweden beside sv_SE, and "module" means a Go package here
|
||||||
|
- Shape: `reference.go:126` `withDefaultModules`: one concept spelled reads, defaultModules and "read by default", the last defined only in the README
|
||||||
|
- Locality: `fejkdata.go:114` `New`: four call sites apply loadError, withDefaultModules and noDataNote in a fixed order, offset by the hidden shipped source being gone
|
||||||
|
|
||||||
|
## 2026-10-11T10:44:13Z, PR #203 at 36b5062, against bf1becb
|
||||||
|
|
||||||
|
Ruling: worse
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | worse | worse | worse | same | worse |
|
||||||
|
| Inherited architect | worse | worse | same | worse | worse |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- Locality, Shape, Overall: `index.go:110` `loadReached`: a reads parameter runs through four layers only to add a note to an error, and reads already means a template's reads
|
||||||
|
- Navigation: `internal/datafiles/datafiles.go:36` `Source.Mounted`: a shipped module's broken file is labelled geo/SE/address.json, a path that no longer exists
|
||||||
|
- Locality: `internal/datafiles/datafiles.go:77` `Source.Walk`: Walk adds the mount and Load removes it, two forms of a folder path in one type
|
||||||
|
- Locality: `fejkdata.go:68` `WithDataFS`: the shipped data no longer loads silently, a partial offset
|
||||||
|
- Shape: `reference.go:127` `withMissingModules`: a typo in manifest-less data still gets a note about modules
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `index.go:110` `loadReached`: reads threaded through six signatures for one error note
|
||||||
|
- Navigation: `internal/datafiles/datafiles.go:36` `Source.Mounted`: errors name sv_SE/address.json, a path that does not exist, where the base named the real data/sv_SE/address.json
|
||||||
|
- Navigation, Self-sufficiency: `data/svse/.fejkdata.json:26` `namespace`: a namespace no longer mirrors its folder, so the path alone no longer finds the file
|
||||||
|
- Self-sufficiency: `internal/datafiles/manifest.go:23` `Manifest.Reads`: its comment and withMissingModules' only make sense after the README
|
||||||
|
- Locality: `internal/datafiles/datafiles.go:77` `Source.Walk`: the mount is translated in opposite directions by Walk and Load
|
||||||
|
|
||||||
|
## 2026-10-11T11:30:01Z, PR #203 at 2a4694b, against bf1becb
|
||||||
|
|
||||||
|
Ruling: worse
|
||||||
|
|
||||||
|
Fell: Navigation. Under the floor of 7.0: Navigation 6.88, so the same ruling counts as worse.
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | worse | better | same | same | same |
|
||||||
|
| Inherited architect | worse | same | same | same | same |
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Junior A | 7 | 5.5 | 6 | 6 | 6 |
|
||||||
|
| Mid A | 7 | 5.5 | 6.5 | 6 | 6 |
|
||||||
|
| Maintainability senior | 7 | 6 | 6.5 | 6.5 | 6.5 |
|
||||||
|
| Inherited architect | 6.5 | 5.5 | 5.5 | 6.5 | 6 |
|
||||||
|
| Mean | 6.88 | 5.63 | 6.13 | 6.25 | 6.13 |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- Locality: `fejkdata.go:66` `WithDataFS`: every module is named at the call site, where the base silently started each generator from a package-level embed
|
||||||
|
- Navigation: `data/geose/fs.go:1` `geose`: a dotfile manifest decides the namespace and the folder follows a lowered, no-separator rule, one translation step from a failing path to its file
|
||||||
|
- Navigation: `internal/datafiles/datafiles.go:57` `Source.labelled`: a shipped file's error prints sv_SE (address.json), where the base printed the openable data/sv_SE/address.json
|
||||||
|
- Locality: `data.go:27` `addSource`: where a category lands depends on a field in a hidden file, a partial offset
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- Navigation: `data/svse/fs.go:1` `svse`: the step from a dot path to a folder goes through the lowered-namespace rule, stated only in the README and each package doc
|
||||||
|
- Navigation: `internal/datafiles/datafiles.go:57` `labelled`: errors name sv_SE (address.json), not a path that opens as written
|
||||||
|
- Locality: `fejkdata.go:99` `New`: New loads exactly its options, cancelled by `data/svse/fs.go:1`'s dependency on geose that nothing in Go expresses
|
||||||
|
|
||||||
|
## 2026-10-11T13:08:34Z, PR #203 at fb2c809, against bf1becb
|
||||||
|
|
||||||
|
Ruling: better
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | same | better | same | same | better |
|
||||||
|
| Inherited architect | better | better | same | better | better |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `fejkdata.go:66` `WithDataFS`: the caller names every module and New refuses no data naming the fix, where the base prepended a package-level shippedSource behind config{shipped: true}
|
||||||
|
- Locality: `data/sv_SE/fs.go:19` `Modules`: sv_SE reading geo.SE is a Go import with a doc line, not only a template reference inside address.json
|
||||||
|
- Shape: `internal/datafiles/mount.go:13` `mounted`: a hand-built virtual fs.FS replaces a base string, balanced by Source losing a field and Walk, Load and Manifest turning path-agnostic
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- Locality, Navigation, Overall: `data/sv_SE/fs.go:19` `Modules`: a locale reading its country's places is a Go import, one hop in code where the base needed the README or a trace through JSON references
|
||||||
|
- Locality, Self-sufficiency: `fejkdata.go:66` `WithDataFS`: every source is named at the call site, and leaving them all out gives an error that spells the fix
|
||||||
|
- Navigation: `data/modules.go:17` `Modules`: shipped data has a Go home inside data/, each module's manifest beside its files
|
||||||
|
- Shape: `internal/datafiles/mount.go:13` `mounted`: seven small types fake the folders above the mount, balanced by Source keeping one form and truthful per-module packages
|
||||||
|
- Self-sufficiency: `data/sv_SE/fs.go:1` `svse`: each module says in code which namespace it supplies and what it reads
|
||||||
|
|
||||||
|
## 2026-10-11T13:18:20Z, PR #203 at 3d4131b, against bf1becb
|
||||||
|
|
||||||
|
Ruling: worse
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | same | better | worse | same | same |
|
||||||
|
| Inherited architect | same | worse | worse | same | worse |
|
||||||
|
| Maintainability senior | same | worse | same | same | worse |
|
||||||
|
| Junior A | same | worse | worse | better | worse |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- Shape: `internal/datafiles/mount.go:13` `mounted`: a hand-built virtual fs.FS of five types replaces a base string joined in dirPath
|
||||||
|
- Locality, Shape: `internal/datafiles/datafiles.go:34` `Source.Mounted`: one field drives file reads, error labels and the manifest read, and a second Mounted call does nothing
|
||||||
|
- Locality: `data/sv_SE/fs.go:19` `Modules`: a locale reading its country's places is a Go import with a package comment
|
||||||
|
- Locality: `fejkdata.go:99` `New`: every source is named at the call site, where the base loaded shipped data implicitly
|
||||||
|
- Shape: `data/modules.go:17` `Modules`: six packages, five near-identical boilerplate
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- Locality, Shape, Overall: `internal/datafiles/mount.go:13` `mounted`: a fake fs.FS of five types with made-up folders above the mount, where the base had a base string; where it rots first
|
||||||
|
- Locality: `internal/datafiles/datafiles.go:34` `Source.Mounted`: the mount point kept in Source.at and mounted.at, and labelled undoes it through underMount
|
||||||
|
- Locality, Self-sufficiency: `index.go:22` `placeIndex`: index keys carry the full namespaced path, which only the mount maps back, stated nowhere at either site
|
||||||
|
- Locality: `fejkdata.go:41` `config.given`: every source explicit and errNoData names the fix, a partial offset
|
||||||
|
- Navigation: `data/modules.go:14` `go:generate`: the manifest generator lives in a test of the parent package
|
||||||
|
|
||||||
|
Maintainability senior, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `internal/datafiles/mount.go:13` `mounted`: a fake fs.FS inventing the folders above the mount, so tracing why a file opens means following three paths in Open, where the base prefixed one base string
|
||||||
|
- Locality: `internal/datafiles/datafiles.go:56` `labelled`: one field serves the manifest read, file opens and error labels, and labelled undoes the mount through code in mount.go
|
||||||
|
- Locality: `fejkdata.go:99` `New`: the hidden shipped default is gone, a partial offset
|
||||||
|
|
||||||
|
Junior A, decided by:
|
||||||
|
|
||||||
|
- Shape, Locality, Overall: `internal/datafiles/mount.go:13` `mounted`: about 130 lines and five types fake folders so a module's root shows up under its namespace, where prefixing Category.Folders looks like the obvious alternative
|
||||||
|
- Locality: `internal/datafiles/datafiles.go:34` `Source.Mounted`: at changes both fsys and how errors label files, and a second Mounted call does nothing silently
|
||||||
|
- Self-sufficiency: `data/sv_SE/fs.go:1` `svse`: each package doc states its namespace and what it reads, and Modules names exactly what it pulls in
|
||||||
|
- Self-sufficiency: `fejkdata.go:79` `errNoData`: the error names the call that loads the shipped data
|
||||||
|
|
||||||
|
## 2026-10-11T18:50:05Z, PR #209 at fbe2fde, against d157e45
|
||||||
|
|
||||||
|
Ruling: better
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | same | better | same | same | better |
|
||||||
|
| Inherited architect | same | better | better | same | better |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `cli/cli.go:318` `options`: the modules a run loads come in through Main and run, where the base fetched data.Modules() inside the flag handling
|
||||||
|
- Locality: `internal/datafiles/datafiles.go:56` `labelled`: reads Source.at, where the base type-asserted the FS to recover the mount point
|
||||||
|
- Locality, Shape: `internal/datafiles/mount.go:11` `mounted`: two types and one helper, no hidden read flag, where the base held five
|
||||||
|
- Locality: `internal/datafiles/datafiles.go:34` `Mounted`: the mount point is stored twice, and a second call is silently ignored
|
||||||
|
- Shape: `cli/cli.go:104` `no-shipped-data`: in a custom CLI the flag drops whatever Main was given, which only Main's doc says
|
||||||
|
- Shape, Self-sufficiency: `data-import/geo-us.py:219` `main`: the street share is a bare positional float, and places is rebound partway
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- Shape, Locality: `internal/datafiles/mount.go:8` `mounted`: Open, ReadFile and ReadDir over one helper, where the base faked a tree in five types
|
||||||
|
- Locality: `internal/datafiles/manifest.go:53` `ManifestPath`: reads the explicit Source.at, though at is stored twice and only Mounted keeps them in step
|
||||||
|
- Locality: `cli/cli.go:318` `options`: the shipped set comes in through Main, run and options, where the base fixed it deep in the flag handling
|
||||||
|
- Shape: `cli/cli.go:104` `no-shipped-data`: once Main takes any modules, the flag's name promises less than it does
|
||||||
|
- Shape: `data-import/geo-us.py:211` `main`: places is rebound from every incorporated place to the shipped ones
|
||||||
|
- Locality: `cli/cli.go:651` `moduleVersion`: a pure function over the build info picks fejkdata's version by path
|
||||||
|
|
||||||
|
## 2026-10-11T19:02:27Z, PR #209 at 69af4c1, against d157e45
|
||||||
|
|
||||||
|
Ruling: better
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | same | better | better | same | better |
|
||||||
|
| Inherited architect | same | better | better | same | better |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- Shape, Locality: `internal/datafiles/mount.go:11` `mounted`: serves Open, ReadFile and ReadDir with one stateless entry type, where the base faked a full tree with a read flag
|
||||||
|
- Locality: `internal/datafiles/datafiles.go:18` `Source`: the mount point is a field, where labelled and ManifestPath type-asserted the FS; a second Mounted is silently ignored
|
||||||
|
- Locality: `cli/cli.go:321` `invocation.options`: the modules and the command name arrive as parameters, where the base called data.Modules() in the flag logic
|
||||||
|
- Shape: `cli/cli.go:678` `versionLine`: the CLI is its own package and main is one line, partly offset by the version line's branches and the COMMAND placeholder
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- Shape, Locality: `internal/datafiles/mount.go:11` `mounted`: two types and one pure underMount, where the base faked an fs.File tree in five types with a read cursor
|
||||||
|
- Locality: `internal/datafiles/manifest.go:53` `ManifestPath`: the manifest reads through s.at whether called before or after Mounted, removing the base's order dependence
|
||||||
|
- Locality: `internal/datafiles/datafiles.go:34` `Mounted`: a second call is a silent no-op and at lives twice; where it rots first
|
||||||
|
- Shape: `cmd/fejkdata/main.go:12` `main`: the binary is one line over a library package, where the base bound the CLI to data.Modules() inside the flag handling
|
||||||
|
- Locality: `cli/cli.go:667` `report`: rewrites the library's prefix inside the joined error text, a string-level contract with another package
|
||||||
|
|
||||||
|
## 2026-10-12T16:29:54Z, PR #216 at 52e0304, against 90ffcff
|
||||||
|
|
||||||
|
Ruling: better
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | same | better | better | better | better |
|
||||||
|
| Inherited architect | same | better | same | same | better |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `fejkdata.go:101` `New`: the seed is plain input, where the base drew it from crypto/rand through the swappable package-level randomBytes and drawSeed
|
||||||
|
- Self-sufficiency, Shape: `fejkdata.go:129` `config.misuse`: one named unit collects every option mistake, and errNoSeed tells the caller what to write
|
||||||
|
- Shape: `fejkdata.go:40` `config.dataGiven`: the name says what it tracks now that seeded sits beside it
|
||||||
|
- Self-sufficiency: `cli/cli.go:26` `usage`: --help states the --seed default and stays true for a custom CLI
|
||||||
|
- Self-sufficiency: `internal/jsonvalue/jsonvalue.go:29` `located`: a JSON syntax error names its line and column
|
||||||
|
- Shape: `record.go:216` `errChoiceRecord`: an inline errors.New became a named sentinel FakeRecord treats alike
|
||||||
|
- Locality: `Dockerfile:14` `release-tooling`: a /tested sentinel copy is an indirection, though its comment explains it
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `fejkdata.go:101` `New`: the seed is a required input and the package-level randomBytes global is gone; New reads straight through options, misuse, load, build
|
||||||
|
- Locality: `fejkdata.go:129` `misuse`: the base's two early returns become one unit joining every option mistake before any data loads
|
||||||
|
- Locality: `cli/cli.go:593` `run`: the CLI's fallback seed is a systemSeed parameter drawn once in Main, so everything that changes the output arrives through run's signature
|
||||||
|
|
||||||
|
## 2026-10-12T16:30:06Z, PR #216 at 52e0304
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Junior A | 7 | 5.5 | 6.5 | 6 | 6 |
|
||||||
|
| Mid A | 7 | 6 | 6.5 | 6.5 | 6.5 |
|
||||||
|
| Maintainability senior | 7 | 5.5 | 6.5 | 6.5 | 6 |
|
||||||
|
| Inherited architect | 7 | 6 | 6 | 7 | 6.5 |
|
||||||
|
| Mean | 7.00 | 5.75 | 6.38 | 6.50 | 6.25 |
|
||||||
|
|
||||||
|
Junior A, hardest first:
|
||||||
|
|
||||||
|
- `pick.go:179` `readName`: six pieces of state, with keptInPick's decision made at load through addressedKeys
|
||||||
|
- `path.go:190` `drawSteps`: climbed carries state between steps and pins swap mid-loop
|
||||||
|
- `table.go:253` `table.route`: six-field tableRoute plus fromRow and pinnedBelow held at once
|
||||||
|
- `node.go:65` `template`: filled in four phases by different files, with a change-together set
|
||||||
|
- `AGENTS.md:109` `misc.loglevel`: the decision title says keyed by keyword, the JSON keys by code
|
||||||
|
|
||||||
|
Mid A, hardest first:
|
||||||
|
|
||||||
|
- `pick.go:9` `pickKey`: keys built at load must match keys built at render, with heap-escape rules mixed in
|
||||||
|
- `render.go:146` `readField`: four read paths, and the name undersells it
|
||||||
|
- `path.go:190` `drawSteps`: order-dependent climbed and stepDownPins
|
||||||
|
- `datatype.go:230` `columnReadOf`: three spellings of a column read, with a four-function change-together contract
|
||||||
|
- `AGENTS.md:109` `misc.loglevel`: the decision title contradicts the JSON
|
||||||
|
|
||||||
|
Maintainability senior, hardest first:
|
||||||
|
|
||||||
|
- `render.go:146` `readField`: choosing named, kept or fresh needs env.pick, pickAt, under, levels and addressed at once
|
||||||
|
- `env.go:71` `enter`: frames pushed here must be found by frameOf later, order dependence across three files
|
||||||
|
- `path.go:190` `drawSteps`: climbed carries state from stepParent to the next stepChild
|
||||||
|
- `path.go:145` `compiledPath.walkEvery`: a capacity-clipping alias with no comment on why it is safe
|
||||||
|
- `index.go:109` `loadReached`: lazy loading mutates the shared tree and rolls back with putBack
|
||||||
|
|
||||||
|
Inherited architect, would restructure first:
|
||||||
|
|
||||||
|
- `pick.go:371` `keptInPick`: string pickKeys rebuilt at render must match addressedKeys built at load, and a miss silently draws afresh
|
||||||
|
- `node.go:253` `template`: one struct for six concepts, filled in four stages by four files
|
||||||
|
- `calc.go:219` `checkCalcFields`: calc lives in five places, and the root calc.go holds only load fences
|
||||||
|
- `datatype.go:222` `columnRead`: column-read resolution sits in datatype.go
|
||||||
|
- `cli/cli.go:505` `isPath`: copies the path grammar, the CLI's first rot point
|
||||||
|
|
||||||
|
## 2026-10-13T22:09:02Z, PR #220 at 909ddc5, against 891323d
|
||||||
|
|
||||||
|
Ruling: same
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | better | worse | same | same | better |
|
||||||
|
| Inherited architect | better | worse | same | same | same |
|
||||||
|
| Maintainability senior | better | worse | same | same | same |
|
||||||
|
| Junior A | better | worse | same | better | better |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- Navigation: `README.md:32` `table of contents` and `README.md:118` `### What stdin renders`: a link row reaches every top-level section, and stdin's byte rules sit under a heading of their own instead of an unheaded block between the CLI examples and the flag table.
|
||||||
|
- Navigation: `README.md:1171` `Layout` row for `release-tooling/` and `README.md:1277` `Development`: a packaging failure leads straight to `packages.yml` or `package-repos.Dockerfile`.
|
||||||
|
- Locality: `release-tooling/goreleaser.yaml:60` `isEnvSet "OLDER_TAG"` and `release-tooling/goreleaser.yaml:61` `dockerfile: ../tooling/release-tooling/image.Dockerfile`: the config assumes the workflow's src/ and tooling/ checkouts, and OLDER_TAG is set by a shell step in `packages.yml:72` with no note where it is read.
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- Locality: `release-tooling/goreleaser.yaml:60` `dockers_v2` tags: whether the image gets `latest` hangs on an env var only `.github/workflows/packages.yml:72` `Channels holding one version` sets, and that step's `--skip` keeps the one-version rule in a second file by a second mechanism.
|
||||||
|
- Locality: `release-tooling/goreleaser.yaml:6` `dist: ../dist`: the config holds only under the two-checkout layout of `packages.yml:42`, and the `snapshot` stage of `package-repos.Dockerfile:5` works only because it skips docker.
|
||||||
|
- Locality: `.github/workflows/test.yml:35` `Relevant changes`: the packaging path list is restated in README.md:1222 and deferred to by AGENTS.md:3, and the copies already differ ("Go code" against `'*.go' go.mod go.sum`).
|
||||||
|
- Navigation: `README.md:32` table of contents, `README.md:36` `## Install` and `README.md:118` `### What stdin renders`: a symptom such as a missing trailing newline now leads to a heading.
|
||||||
|
- Navigation: `README.md:1277` release paragraph: one paragraph traces the changelog heading, `publish_release.py`, the tag output and `packages.yml`, with the secrets and re-packaging.
|
||||||
|
|
||||||
|
Maintainability senior, decided by:
|
||||||
|
|
||||||
|
- Locality: `release-tooling/goreleaser.yaml:6` `dist` and `release-tooling/goreleaser.yaml:61` `dockers_v2.dockerfile`: `../dist` and `../tooling/...` resolve only under the two sibling checkouts `packages.yml:42` sets up, and the header explains `dist` but not `../tooling/`.
|
||||||
|
- Locality: `.github/workflows/packages.yml:72` `Channels holding one version` and `release-tooling/goreleaser.yaml:60` `latest` tag template: OLDER_TAG is set in one file and read through `isEnvSet` in another, and the same step handles an older tag a second way through `--skip`.
|
||||||
|
- Locality: `Dockerfile:18` `packaging`, `release-tooling/package-repos.Dockerfile:5` `snapshot` and `.github/workflows/packages.yml:82` `goreleaser-action`: one GoReleaser version pinned in three files, and `packages.yml:56` reads the Go version by `sed` over the Dockerfile's ARG line.
|
||||||
|
- Locality: `.github/workflows/test.yml:35` `Relevant changes`: the packaging path list is written here and again as prose in README.md:1222, which AGENTS.md:3 points the gate at.
|
||||||
|
- Navigation: `README.md:32` table of contents and `README.md:118` `### What stdin renders`: the stdin byte rules have a heading to jump to.
|
||||||
|
- Navigation: `README.md:1171` `## Layout` and `README.md:1277` `## Development`: a cold reader goes from a wrong apt version to `packages.yml`, `package-repos.Dockerfile` and `check-deb`.
|
||||||
|
|
||||||
|
Junior A, decided by:
|
||||||
|
|
||||||
|
- Navigation: `README.md:32` table of contents and `README.md:36` `## Install`: a reader with a symptom jumps to a named section, and the stdin byte rules sit under `README.md:118` `### What stdin renders`.
|
||||||
|
- Navigation, Self-sufficiency: `README.md:1171` `release-tooling/` in Layout and `README.md:1222` `## Development`: a failed install or Packages run leads to `release-tooling/package-repos.Dockerfile`, and each secret is named.
|
||||||
|
- Locality: `.github/workflows/packages.yml:72` `Channels holding one version` and `release-tooling/goreleaser.yaml:60` `isEnvSet "OLDER_TAG"`: whether the image gets `latest` is decided by a variable set in another file, with no note where it is read.
|
||||||
|
- Locality: `Dockerfile:18` `packaging`, `release-tooling/package-repos.Dockerfile:5` `snapshot` and `.github/workflows/packages.yml:82` `version`: one GoReleaser version pinned in three places, tied only by renovate.json.
|
||||||
|
- Self-sufficiency: `release-tooling/goreleaser.yaml:1`, `.github/workflows/packages.yml:48` and `release-tooling/package-repos.Dockerfile:1`: each non-obvious choice is explained where it is made.
|
||||||
|
- Self-sufficiency: `doc.go:5` package doc: the namespace sentence now has its subject.
|
||||||
|
|
||||||
|
## 2026-10-14T09:22:16Z, PR #222 at e492511, against c3cc251
|
||||||
|
|
||||||
|
Ruling: same
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | same | same | same | same | same |
|
||||||
|
| Inherited architect | same | same | same | same | same |
|
||||||
|
|
||||||
|
## 2026-10-14T11:34:34Z, PR #225 at f734c9e, against f90b4d6
|
||||||
|
|
||||||
|
Ruling: worse
|
||||||
|
|
||||||
|
| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| Mid A | same | worse | same | same | same |
|
||||||
|
| Inherited architect | same | worse | same | same | worse |
|
||||||
|
| Maintainability senior | same | worse | same | worse | worse |
|
||||||
|
| Junior A | same | worse | same | worse | worse |
|
||||||
|
|
||||||
|
Mid A, decided by:
|
||||||
|
|
||||||
|
- Locality: `Dockerfile:26` `vulnerability-filing`: the filer's test reads the real todo.md inside docker build ., so a plan edit can turn the Go gate red
|
||||||
|
- Locality: `vulnerability-filing/file_vulnerabilities.py:24` `ROW`: the filer splices todo.md as text, assuming its row shape, the Next ID marker and the Details heading
|
||||||
|
- Locality: `.github/workflows/test.yml:106` `release`: a release now also waits on vuln.go.dev's answer, visible only in a needs list
|
||||||
|
|
||||||
|
Inherited architect, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `Dockerfile:26` `vulnerability-filing`: the plan is copied into the merge-gate build, so a planning edit that changes the table's shape fails docker build .
|
||||||
|
- Locality: `vulnerability-filing/file_vulnerabilities.py:101` `file_items`: edits todo.md as text; a missing Next ID crashes with exit 1, which reads like a vulnerability already planned
|
||||||
|
- Locality: `.github/workflows/test.yml:69` `vulnerabilities`: whether a release happens depends on vuln.go.dev that day, and every PR shows a red job while one is known
|
||||||
|
|
||||||
|
Maintainability senior, decided by:
|
||||||
|
|
||||||
|
- Locality: `vulnerability-filing/file_vulnerabilities.py:101` `file_items`: finds the release by an unnamed regex group, never checks the Next ID match, and splices Details by a replace that does nothing when the heading is missing
|
||||||
|
- Locality, Self-sufficiency: `.github/workflows/test.yml:83` `File each one in todo.md`: one exit code holds the release and triggers the proposal, which must tell filed, planned and crashed apart by a git diff
|
||||||
|
- Locality: `Dockerfile:26` `vulnerability-filing`: copies todo.md into the gate build, and the path filter lists todo.md
|
||||||
|
- Self-sufficiency: `README.md:1213` "Development": says the build runs the release tooling's tests and leaves out the filer's
|
||||||
|
- Self-sufficiency: `vulnerability-filing/file_vulnerabilities.py:2` module docstring: builds on rank_todo.py, which the repository does not hold
|
||||||
|
|
||||||
|
Junior A, decided by:
|
||||||
|
|
||||||
|
- Locality, Overall: `Dockerfile:26` `vulnerability-filing`: the Go merge gate copies todo.md and files an item into it, so editing the plan can turn the build red
|
||||||
|
- Locality: `.github/workflows/test.yml:106` `release`: whether a push to main releases also depends on vuln.go.dev's answer that day
|
||||||
|
- Self-sufficiency: `vulnerability-filing/file_vulnerabilities.py:24` `ROW`: the regex and the Details splice encode todo.md's layout, which the script never shows
|
||||||
|
- Self-sufficiency: `README.md:1213` "Development": the gate paragraph leaves out the filer's tests while AGENTS.md now defers to it
|
||||||
+1012
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,637 @@
|
|||||||
|
# Reading cost
|
||||||
|
|
||||||
|
## Reading cost, chunk aedb337e0fa8, finished 2026-10-04T21:47:53Z
|
||||||
|
|
||||||
|
68314 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `README.md` 9
|
||||||
|
- `todo.md` 6
|
||||||
|
- `compose.yaml` 3
|
||||||
|
- `data-import/territory.py` 2
|
||||||
|
- `data_test.go` 2
|
||||||
|
- `shipped_data_test.go` 2
|
||||||
|
- `AGENTS.md` 1
|
||||||
|
- `CHANGELOG.md` 1
|
||||||
|
- `data-import/tsv.py` 1
|
||||||
|
- `data/en_US/phone.json` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 501b569f8d46, finished 2026-10-04T22:23:51Z
|
||||||
|
|
||||||
|
118930 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `docs/decisions.md` 8
|
||||||
|
- `README.md` 6
|
||||||
|
- `table_test.go` 4
|
||||||
|
- `render.go` 3
|
||||||
|
- `todo.md` 3
|
||||||
|
- `CHANGELOG.md` 2
|
||||||
|
- `pick.go` 2
|
||||||
|
- `table.go` 2
|
||||||
|
- `AGENTS.md` 1
|
||||||
|
- `fejkdata.go` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 2dd5dac8fb7d, finished 2026-10-04T22:50:50Z
|
||||||
|
|
||||||
|
93128 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `README.md` 6
|
||||||
|
- `data_test.go` 4
|
||||||
|
- `todo.md` 3
|
||||||
|
- `format_stability_test.go` 2
|
||||||
|
- `CHANGELOG.md` 1
|
||||||
|
- `compose.yaml` 1
|
||||||
|
- `data/en_US/email.json` 1
|
||||||
|
- `data/en_US/first-name.tsv` 1
|
||||||
|
- `data/en_US/sex.tsv` 1
|
||||||
|
- `data/en_US/title.tsv` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 305e76235620, finished 2026-10-05T06:43:56Z
|
||||||
|
|
||||||
|
79628 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `README.md` 5
|
||||||
|
- `todo.md` 5
|
||||||
|
- `format.go` 4
|
||||||
|
- `valueproof.go` 4
|
||||||
|
- `calc_test.go` 3
|
||||||
|
- `datatype_test.go` 2
|
||||||
|
- `name.go` 2
|
||||||
|
- `name_test.go` 2
|
||||||
|
- `record_test.go` 2
|
||||||
|
- `CHANGELOG.md` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 5d443718e378, finished 2026-10-05T08:24:10Z
|
||||||
|
|
||||||
|
366332 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `table_test.go` 17
|
||||||
|
- `table.go` 9
|
||||||
|
- `struct.go` 8
|
||||||
|
- `README.md` 7
|
||||||
|
- `docs/decisions.md` 7
|
||||||
|
- `record.go` 5
|
||||||
|
- `calc.go` 4
|
||||||
|
- `data_test.go` 4
|
||||||
|
- `reference.go` 4
|
||||||
|
- `render.go` 4
|
||||||
|
|
||||||
|
## Reading cost, chunk 95fbdb646b25, finished 2026-10-05T09:41:42Z
|
||||||
|
|
||||||
|
705270 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `docs/decisions.md` 16
|
||||||
|
- `todo.md` 16
|
||||||
|
- `README.md` 10
|
||||||
|
- `render.go` 5
|
||||||
|
- `table.go` 5
|
||||||
|
- `format.go` 4
|
||||||
|
- `AGENTS.md` 3
|
||||||
|
- `calc.go` 3
|
||||||
|
- `data.go` 3
|
||||||
|
- `fejkdata.go` 3
|
||||||
|
|
||||||
|
## Reading cost, chunk 790a992c1467, finished 2026-10-05T10:04:02Z
|
||||||
|
|
||||||
|
59846 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `todo.md` 6
|
||||||
|
- `CHANGELOG.md` 2
|
||||||
|
- `README.md` 2
|
||||||
|
- `docs/decisions.md` 2
|
||||||
|
- `fejkdata.go` 2
|
||||||
|
- `fejkdata_test.go` 2
|
||||||
|
- `go.mod` 2
|
||||||
|
- `.dockerignore` 1
|
||||||
|
- `builtins_test.go` 1
|
||||||
|
- `cmd/fejkdata/main.go` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 2ef0d4af8d44, finished 2026-10-05T13:19:29Z
|
||||||
|
|
||||||
|
98933 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `format_test.go` 5
|
||||||
|
- `todo.md` 5
|
||||||
|
- `calc.go` 4
|
||||||
|
- `calc_test.go` 4
|
||||||
|
- `format.go` 4
|
||||||
|
- `path.go` 3
|
||||||
|
- `reference.go` 3
|
||||||
|
- `README.md` 2
|
||||||
|
- `builtins_test.go` 2
|
||||||
|
- `inline.go` 2
|
||||||
|
|
||||||
|
## Reading cost, chunk ebcf577f6435, finished 2026-10-05T19:01:32Z
|
||||||
|
|
||||||
|
129910 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `builtins_test.go` 7
|
||||||
|
- `todo.md` 5
|
||||||
|
- `format.go` 4
|
||||||
|
- `calc_test.go` 2
|
||||||
|
- `docs/decisions.md` 2
|
||||||
|
- `fejkdata.go` 2
|
||||||
|
- `reference.go` 2
|
||||||
|
- `CHANGELOG.md` 1
|
||||||
|
- `README.md` 1
|
||||||
|
- `calc.go` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 4f0407382a32, finished 2026-10-05T19:22:15Z
|
||||||
|
|
||||||
|
91864 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `README.md` 4
|
||||||
|
- `table_test.go` 4
|
||||||
|
- `data.go` 3
|
||||||
|
- `todo.md` 3
|
||||||
|
- `graph.go` 2
|
||||||
|
- `node.go` 2
|
||||||
|
- `CHANGELOG.md` 1
|
||||||
|
- `compose.yaml` 1
|
||||||
|
- `fejkdata.go` 1
|
||||||
|
- `inline.go` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk f695f44a8702, finished 2026-10-05T20:10:33Z
|
||||||
|
|
||||||
|
98023 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `todo.md` 6
|
||||||
|
- `path_test.go` 4
|
||||||
|
- `README.md` 3
|
||||||
|
- `docs/decisions.md` 3
|
||||||
|
- `imports_test.go` 2
|
||||||
|
- `path.go` 2
|
||||||
|
- `reference_test.go` 2
|
||||||
|
- `table.go` 2
|
||||||
|
- `AGENTS.md` 1
|
||||||
|
- `CHANGELOG.md` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 958a19b8d709, finished 2026-10-05T20:49:27Z
|
||||||
|
|
||||||
|
115125 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `name_test.go` 5
|
||||||
|
- `todo.md` 5
|
||||||
|
- `shape_test.go` 3
|
||||||
|
- `README.md` 2
|
||||||
|
- `graph.go` 2
|
||||||
|
- `inline.go` 2
|
||||||
|
- `node.go` 2
|
||||||
|
- `record_test.go` 2
|
||||||
|
- `struct.go` 2
|
||||||
|
- `CHANGELOG.md` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 35ed2f5f3f4c, finished 2026-10-05T21:03:20Z
|
||||||
|
|
||||||
|
53085 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `AGENTS.md` 3
|
||||||
|
- `README.md` 3
|
||||||
|
- `todo.md` 3
|
||||||
|
- `.gitignore` 1
|
||||||
|
- `CHANGELOG.md` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk ee081b4f8c13, finished 2026-10-05T21:20:27Z
|
||||||
|
|
||||||
|
67696 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `todo.md` 3
|
||||||
|
- `pick_test.go` 2
|
||||||
|
- `render.go` 2
|
||||||
|
- `table_test.go` 2
|
||||||
|
- `CHANGELOG.md` 1
|
||||||
|
- `README.md` 1
|
||||||
|
- `data/geo/SE/address.json` 1
|
||||||
|
- `data/sv_SE/address.json` 1
|
||||||
|
- `docs/decisions.md` 1
|
||||||
|
- `pick.go` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 4c8a3bfa0fc7, finished 2026-10-06T22:46:06Z
|
||||||
|
|
||||||
|
75959 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `todo.md` 5
|
||||||
|
- `README.md` 2
|
||||||
|
- `inline.go` 2
|
||||||
|
- `namefence.go` 2
|
||||||
|
- `path.go` 2
|
||||||
|
- `pick_test.go` 2
|
||||||
|
- `record.go` 2
|
||||||
|
- `AGENTS.md` 1
|
||||||
|
- `CHANGELOG.md` 1
|
||||||
|
- `compose.yaml` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 3fe488db154a, finished 2026-10-06T23:05:09Z
|
||||||
|
|
||||||
|
129293 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `todo.md` 9
|
||||||
|
- `README.md` 6
|
||||||
|
- `format.go` 6
|
||||||
|
- `docs/decisions.md` 5
|
||||||
|
- `record.go` 5
|
||||||
|
- `inline.go` 4
|
||||||
|
- `render.go` 4
|
||||||
|
- `shipped.go` 4
|
||||||
|
- `CHANGELOG.md` 3
|
||||||
|
- `data.go` 3
|
||||||
|
|
||||||
|
## Reading cost, chunk 9dbdacef756b, finished 2026-10-06T23:24:42Z
|
||||||
|
|
||||||
|
79101 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `README.md` 5
|
||||||
|
- `format.go` 5
|
||||||
|
- `todo.md` 5
|
||||||
|
- `path.go` 3
|
||||||
|
- `docs/decisions.md` 2
|
||||||
|
- `env.go` 2
|
||||||
|
- `pick.go` 2
|
||||||
|
- `table.go` 2
|
||||||
|
- `AGENTS.md` 1
|
||||||
|
- `CHANGELOG.md` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 8d30d77fba26, finished 2026-10-06T23:45:50Z
|
||||||
|
|
||||||
|
2986167 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `README.md` 24
|
||||||
|
- `AGENTS.md` 18
|
||||||
|
- `CHANGELOG.md` 18
|
||||||
|
- `data-import/geo-se.py` 17
|
||||||
|
- `record.go` 17
|
||||||
|
- `cmd/fejkdata/main.go` 16
|
||||||
|
- `struct.go` 16
|
||||||
|
- `testdata/shipped_shape.txt` 16
|
||||||
|
- `.github/workflows/test.yml` 15
|
||||||
|
- `arm.go` 15
|
||||||
|
|
||||||
|
## Reading cost, chunk 0e8a426874fc, finished 2026-10-07T00:04:56Z
|
||||||
|
|
||||||
|
96842 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `README.md` 6
|
||||||
|
- `CHANGELOG.md` 4
|
||||||
|
- `todo.md` 4
|
||||||
|
- `data-import/geo-se.py` 2
|
||||||
|
- `doc.go` 2
|
||||||
|
- `docs/decisions.md` 2
|
||||||
|
- `path.go` 2
|
||||||
|
- `resolve.go` 2
|
||||||
|
- `AGENTS.md` 1
|
||||||
|
- `arm.go` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 2b3baad177ba, finished 2026-10-07T00:27:52Z
|
||||||
|
|
||||||
|
152553 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `README.md` 9
|
||||||
|
- `todo.md` 8
|
||||||
|
- `data-import/geo-us.py` 5
|
||||||
|
- `AGENTS.md` 4
|
||||||
|
- `fejkdata.go` 4
|
||||||
|
- `graph.go` 4
|
||||||
|
- `docs/decisions.md` 3
|
||||||
|
- `name.go` 3
|
||||||
|
- `pick.go` 3
|
||||||
|
- `resolve.go` 3
|
||||||
|
|
||||||
|
## Reading cost, chunk 4e18ae82f3dc, finished 2026-10-09T08:51:06Z
|
||||||
|
|
||||||
|
2852445 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `README.md` 23
|
||||||
|
- `cmd/fejkdata/main.go` 23
|
||||||
|
- `pick.go` 18
|
||||||
|
- `AGENTS.md` 17
|
||||||
|
- `CHANGELOG.md` 16
|
||||||
|
- `compose.yaml` 16
|
||||||
|
- `env.go` 16
|
||||||
|
- `arm.go` 15
|
||||||
|
- `fejkdata.go` 15
|
||||||
|
- `path.go` 15
|
||||||
|
|
||||||
|
## Reading cost, chunk 99b75bd10e88, finished 2026-10-09T09:49:13Z
|
||||||
|
|
||||||
|
93957 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `todo.md` 9
|
||||||
|
- `name.go` 6
|
||||||
|
- `README.md` 5
|
||||||
|
- `graph.go` 4
|
||||||
|
- `env.go` 3
|
||||||
|
- `path.go` 3
|
||||||
|
- `resolve.go` 3
|
||||||
|
- `arm.go` 2
|
||||||
|
- `compose.yaml` 2
|
||||||
|
- `docs/decisions.md` 2
|
||||||
|
|
||||||
|
## Reading cost, chunk 379810cdbae5, finished 2026-10-09T10:01:14Z
|
||||||
|
|
||||||
|
68500 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `render_test.go` 6
|
||||||
|
- `todo.md` 3
|
||||||
|
- `README.md` 2
|
||||||
|
- `docs/decisions.md` 2
|
||||||
|
- `AGENTS.md` 1
|
||||||
|
- `CHANGELOG.md` 1
|
||||||
|
- `doc.go` 1
|
||||||
|
- `doc_test.go` 1
|
||||||
|
- `node.go` 1
|
||||||
|
- `render.go` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 3374028f1e29, finished 2026-10-09T10:35:44Z
|
||||||
|
|
||||||
|
69756 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `todo.md` 6
|
||||||
|
- `README.md` 3
|
||||||
|
- `docs/decisions.md` 3
|
||||||
|
- `CHANGELOG.md` 2
|
||||||
|
- `AGENTS.md` 1
|
||||||
|
- `compose.yaml` 1
|
||||||
|
- `data-import/geo-se.py` 1
|
||||||
|
- `data-import/geo-us.py` 1
|
||||||
|
- `data-import/geo.py` 1
|
||||||
|
- `data-import/tsv.py` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk c484375b01f5, finished 2026-10-09T10:53:55Z
|
||||||
|
|
||||||
|
60588 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `todo.md` 6
|
||||||
|
- `README.md` 2
|
||||||
|
- `docs/decisions.md` 2
|
||||||
|
- `record.go` 2
|
||||||
|
- `render.go` 2
|
||||||
|
- `CHANGELOG.md` 1
|
||||||
|
- `docs/comprehension-history.md` 1
|
||||||
|
- `inline.go` 1
|
||||||
|
- `reference.go` 1
|
||||||
|
- `shipped.go` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 21603e0f3240, finished 2026-10-09T12:30:43Z
|
||||||
|
|
||||||
|
2855972 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `README.md` 25
|
||||||
|
- `CHANGELOG.md` 22
|
||||||
|
- `cmd/fejkdata/main.go` 20
|
||||||
|
- `node.go` 19
|
||||||
|
- `testdata/shipped_shape.txt` 19
|
||||||
|
- `AGENTS.md` 18
|
||||||
|
- `arm.go` 17
|
||||||
|
- `DATA-LICENSES.md` 16
|
||||||
|
- `compose.yaml` 16
|
||||||
|
- `env.go` 16
|
||||||
|
|
||||||
|
## Reading cost, chunk f73b4c751678, finished 2026-10-09T13:02:23Z
|
||||||
|
|
||||||
|
340530 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `todo.md` 15
|
||||||
|
- `docs/decisions.md` 14
|
||||||
|
- `README.md` 4
|
||||||
|
- `.github/workflows/test.yml` 2
|
||||||
|
- `compose.yaml` 2
|
||||||
|
- `fejkdata.go` 2
|
||||||
|
- `go.mod` 2
|
||||||
|
- `release-tooling/publish_release.py` 2
|
||||||
|
- `shipped.go` 2
|
||||||
|
- `shippedindex.go` 2
|
||||||
|
|
||||||
|
## Reading cost, chunk af1b90b1481c, finished 2026-10-09T17:00:45Z
|
||||||
|
|
||||||
|
91452 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `README.md` 7
|
||||||
|
- `todo.md` 5
|
||||||
|
- `cmd/fejkdata/main_test.go` 3
|
||||||
|
- `CHANGELOG.md` 2
|
||||||
|
- `docs/decisions.md` 2
|
||||||
|
- `imports_test.go` 2
|
||||||
|
- `.dockerignore` 1
|
||||||
|
- `.github/workflows/test.yml` 1
|
||||||
|
- `cmd/fejkdata/main.go` 1
|
||||||
|
- `compose.yaml` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 2f68b5245844, finished 2026-10-10T10:55:47Z
|
||||||
|
|
||||||
|
112728 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `README.md` 9
|
||||||
|
- `todo.md` 6
|
||||||
|
- `docs/decisions.md` 5
|
||||||
|
- `shipped_test.go` 2
|
||||||
|
- `CHANGELOG.md` 1
|
||||||
|
- `compose.yaml` 1
|
||||||
|
- `data.go` 1
|
||||||
|
- `fejkdata.go` 1
|
||||||
|
- `inline.go` 1
|
||||||
|
- `internal/datafiles/datafiles.go` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 1c769e839e3a, finished 2026-10-11T14:05:12Z
|
||||||
|
|
||||||
|
200600 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `README.md` 13
|
||||||
|
- `todo.md` 8
|
||||||
|
- `CHANGELOG.md` 6
|
||||||
|
- `cmd/fejkdata/main.go` 6
|
||||||
|
- `docs/decisions.md` 6
|
||||||
|
- `shipped_test.go` 6
|
||||||
|
- `DATA-LICENSES.md` 5
|
||||||
|
- `bench_test.go` 5
|
||||||
|
- `data_test.go` 5
|
||||||
|
- `data.go` 4
|
||||||
|
|
||||||
|
## Reading cost, chunk 8ce3be08e54e, finished 2026-10-11T16:36:06Z
|
||||||
|
|
||||||
|
87725 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `todo.md` 6
|
||||||
|
- `README.md` 4
|
||||||
|
- `data_test.go` 4
|
||||||
|
- `CHANGELOG.md` 2
|
||||||
|
- `docs/decisions.md` 2
|
||||||
|
- `compose.yaml` 1
|
||||||
|
- `data-import/geo-se.py` 1
|
||||||
|
- `data-import/geo-us.py` 1
|
||||||
|
- `data-import/geo.py` 1
|
||||||
|
- `data-import/source.py` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk a8085e6ee71e, finished 2026-10-11T17:14:17Z
|
||||||
|
|
||||||
|
90566 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `data-import/geo-us.py` 6
|
||||||
|
- `README.md` 4
|
||||||
|
- `todo.md` 4
|
||||||
|
- `data-import/geo-se.py` 3
|
||||||
|
- `compose.yaml` 2
|
||||||
|
- `data/geo/US/locality.tsv` 2
|
||||||
|
- `data_test.go` 2
|
||||||
|
- `.gitignore` 1
|
||||||
|
- `AGENTS.md` 1
|
||||||
|
- `CHANGELOG.md` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk c1a0ce6b3d32, finished 2026-10-11T18:41:31Z
|
||||||
|
|
||||||
|
76079 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `todo.md` 10
|
||||||
|
- `docs/decisions.md` 6
|
||||||
|
- `README.md` 4
|
||||||
|
- `AGENTS.md` 2
|
||||||
|
- `data-import/geo-se.py` 2
|
||||||
|
- `CHANGELOG.md` 1
|
||||||
|
- `data-import/geo-us.py` 1
|
||||||
|
- `data-import/geo.py` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 0cd4f425fa6f, finished 2026-10-11T21:08:07Z
|
||||||
|
|
||||||
|
102267 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `README.md` 7
|
||||||
|
- `CHANGELOG.md` 6
|
||||||
|
- `todo.md` 6
|
||||||
|
- `cli/cli_test.go` 4
|
||||||
|
- `docs/decisions.md` 3
|
||||||
|
- `cmd/fejkdata/main_test.go` 2
|
||||||
|
- `AGENTS.md` 1
|
||||||
|
- `cmd/fejkdata/main.go` 1
|
||||||
|
- `compose.yaml` 1
|
||||||
|
- `data/modules.go` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk d67229c6358e, finished 2026-10-11T21:39:58Z
|
||||||
|
|
||||||
|
372546 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `README.md` 30
|
||||||
|
- `todo.md` 10
|
||||||
|
- `cli/cli.go` 5
|
||||||
|
- `cli/cli_test.go` 3
|
||||||
|
- `CHANGELOG.md` 2
|
||||||
|
- `data/sv_SE/.fejkdata.json` 2
|
||||||
|
- `docs/decisions.md` 2
|
||||||
|
- `internal/datafiles/datafiles.go` 2
|
||||||
|
- `record.go` 2
|
||||||
|
- `compose.yaml` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk aca06ea931a8, finished 2026-10-12T15:34:32Z
|
||||||
|
|
||||||
|
78327 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `todo.md` 8
|
||||||
|
- `README.md` 5
|
||||||
|
- `CHANGELOG.md` 1
|
||||||
|
- `DATA-LICENSES.md` 1
|
||||||
|
- `data/misc/car.tsv` 1
|
||||||
|
- `docs/decisions.md` 1
|
||||||
|
- `docs/research/research-sources-se.md` 1
|
||||||
|
- `docs/research/research-sources-world.md` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk 2f1148286c4c, finished 2026-10-12T15:50:50Z
|
||||||
|
|
||||||
|
60817 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `README.md` 4
|
||||||
|
- `todo.md` 3
|
||||||
|
- `CHANGELOG.md` 2
|
||||||
|
- `shipped_data_test.go` 2
|
||||||
|
- `.dockerignore` 1
|
||||||
|
- `.gitignore` 1
|
||||||
|
- `data_test.go` 1
|
||||||
|
- `docs/decisions.md` 1
|
||||||
|
- `release-tooling/publish_release.py` 1
|
||||||
|
- `shipped_test.go` 1
|
||||||
|
|
||||||
|
## Reading cost, chunk f26a17e98714, finished 2026-10-12T16:07:59Z
|
||||||
|
|
||||||
|
78241 new input tokens before the first commit.
|
||||||
|
|
||||||
|
Read most (opens and searches):
|
||||||
|
|
||||||
|
- `README.md` 10
|
||||||
|
- `todo.md` 7
|
||||||
|
- `CHANGELOG.md` 3
|
||||||
|
- `docs/decisions.md` 3
|
||||||
|
- `compose.yaml` 1
|
||||||
|
- `data/en_US/itin.json` 1
|
||||||
|
- `data/misc/timezone.json` 1
|
||||||
|
- `data/misc/timezone.tsv` 1
|
||||||
|
- `go.mod` 1
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
# Population by age band — research 2026-10-04
|
||||||
|
|
||||||
|
Share of the population in four bands, from each country's statistics office, for weighting a drawn birthdate by age. Computed from single-year ages.
|
||||||
|
|
||||||
|
| Country | Date | Total | 0–17 | 18–64 | 65–99 | 100+ | Source |
|
||||||
|
|---|---|---|---|---|---|---|---|
|
||||||
|
| SE | 2024-12-31 | 10,587,710 | 20.36% | 58.80% | 20.81% | 0.028% (2,961) | SCB Statistikdatabasen, table BE0101A/BefolkningNy, CC0 1.0 |
|
||||||
|
| US | 2024-07-01 | 340,110,988 | 21.50% | 60.51% | 17.96% | 0.029% (98,034) | Census Bureau, [nc-est2024-agesex-res.csv](https://www2.census.gov/programs-surveys/popest/datasets/2020-2024/national/asrh/nc-est2024-agesex-res.csv), public domain |
|
||||||
|
|
||||||
|
At 0.03%, a fixture of 10,000 rows holds about three people aged 100 or more.
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
# Swedish (sv_SE) fake-data sources — research 2026-09-17
|
# Swedish (sv_SE) fake-data sources — research 2026-09-17
|
||||||
|
|
||||||
Scope: names, identifiers, phone, company, plates/cars, words, dates/misc, other. Geography excluded.
|
Scope: names, identifiers, phone, company, plates/cars, words, dates/misc, other. Geography excluded.
|
||||||
"Verified" = fetched and read this session. Local copies of inspected files: `/home/lilleman/.claude/jobs/e377c8d4/tmp/`.
|
"Verified" = fetched and read for this research.
|
||||||
|
|
||||||
## Licence summary (embed in MIT repo?)
|
## Licence summary (embed in MIT repo?)
|
||||||
|
|
||||||
@@ -15,7 +15,7 @@ Scope: names, identifiers, phone, company, plates/cars, words, dates/misc, other
|
|||||||
| Bolagsverket/SCB bulk company files | Not stated ("värdefulla datamängder", free, no agreement) | Derive stats only; do not embed raw | Cite |
|
| Bolagsverket/SCB bulk company files | Not stated ("värdefulla datamängder", free, no agreement) | Derive stats only; do not embed raw | Cite |
|
||||||
| Bankinfrastruktur clearing CSV | Not stated; accuracy disclaimer | Facts only; flag | Cite BSAB |
|
| Bankinfrastruktur clearing CSV | Not stated; accuracy disclaimer | Facts only; flag | Cite BSAB |
|
||||||
| Bankgirot PDFs | "© Bankgirocentralen BGC AB. All rights reserved." (Informationsklass: Öppen) | No verbatim copy; facts only | — |
|
| Bankgirot PDFs | "© Bankgirocentralen BGC AB. All rights reserved." (Informationsklass: Öppen) | No verbatim copy; facts only | — |
|
||||||
| PTS numbering plan PDF | Not stated (no licence on pts.se or in PDF) | Facts only; flag | Cite PTS |
|
| PTS numbering plan PDF, pts.se number pages | Not stated (no licence on pts.se or in PDF) | Facts only; flag | Cite PTS |
|
||||||
| Transportstyrelsen blocked combinations | Not stated on page | Facts (a list of 99 codes); flag | Cite |
|
| Transportstyrelsen blocked combinations | Not stated on page | Facts (a list of 99 codes); flag | Cite |
|
||||||
| SALDO morphology (Språkbanken) | CC BY 4.0 | Yes | Cite: Språkbanken (2017) SALDO's morphology, DOI 10.23695/agcm-ny22 |
|
| SALDO morphology (Språkbanken) | CC BY 4.0 | Yes | Cite: Språkbanken (2017) SALDO's morphology, DOI 10.23695/agcm-ny22 |
|
||||||
| Språkbanken word statistics | CC BY 4.0 | Yes | Cite Språkbanken |
|
| Språkbanken word statistics | CC BY 4.0 | Yes | Cite Språkbanken |
|
||||||
@@ -90,6 +90,7 @@ Recommendation: embed SCB 2022 xlsx-derived lists (top-N per sheet with counts)
|
|||||||
## 3. Phone
|
## 3. Phone
|
||||||
|
|
||||||
- Source: PTS "The Swedish numbering plan for telephony according to ITU-T E.164", 2024-01-08 (verified, text extracted): https://pts.se/globalassets/globala-block/nummertillstand/the-swedish-numbering-plan-for-telephony-according-to-itu---2024-01-08.pdf. Swedish "nrplansammanstallning-2026-02-04.pdf" and https://nummer.pts.se/NbrPlanSearch are behind a Radware bot check (both curl and Playwright blocked). No licence stated by PTS anywhere found; pts.se says contact pts@pts.se for data.
|
- Source: PTS "The Swedish numbering plan for telephony according to ITU-T E.164", 2024-01-08 (verified, text extracted): https://pts.se/globalassets/globala-block/nummertillstand/the-swedish-numbering-plan-for-telephony-according-to-itu---2024-01-08.pdf. Swedish "nrplansammanstallning-2026-02-04.pdf" and https://nummer.pts.se/NbrPlanSearch are behind a Radware bot check (both curl and Playwright blocked). No licence stated by PTS anywhere found; pts.se says contact pts@pts.se for data.
|
||||||
|
- Reserved for fiction, PTS "Telefonnummer till böcker och filmer" (verified 2026-09-21; curl and Playwright are Radware-blocked, a rendering fetcher gets through): https://www.pts.se/internet-och-telefoni/telefonnummer-och-adressering/telefonnummer-till-bocker-och-filmer/ — mobile 070-1740605–1740699, geographic 08-46500400–46500499 (Stockholm), 031-3900600–3900699 (Göteborg), 040-6280400–6280499 (Malmö), 0980-319200–319299 (Kiruna). 495 numbers PTS holds back so they reach nobody, which is goal 15's never-reaching path for `sv_SE.phone`. Same licence status as the numbering plan: none stated, facts only, cite PTS. The mobile block fits the shipped `{a} {b} {c}` grouping (`174 06 05`–`99`), as do 031 and 040; Stockholm's 8 subscriber digits and Kiruna's 6 need a format of their own, and 0980 is not among the 20 shipped area codes.
|
||||||
- 264 geographic area codes (PDF text yields 263 "Area code for" rows; one likely split across a page break — flag). Area code = trunk "0" + NDC of 1–3 digits (08 Stockholm, 031 Göteborg, 040 Malmö, 0480 Kalmar). Geographic N(S)N (NDC + subscriber) max 9 digits, min 7 (2-digit NDC) or 8 (3-digit NDC); Stockholm min 7. Each row in the PDF gives NDC, max/min length, "Area code for <name>", so a `{ndc, name}` list can be built from it.
|
- 264 geographic area codes (PDF text yields 263 "Area code for" rows; one likely split across a page break — flag). Area code = trunk "0" + NDC of 1–3 digits (08 Stockholm, 031 Göteborg, 040 Malmö, 0480 Kalmar). Geographic N(S)N (NDC + subscriber) max 9 digits, min 7 (2-digit NDC) or 8 (3-digit NDC); Stockholm min 7. Each row in the PDF gives NDC, max/min length, "Area code for <name>", so a `{ndc, name}` list can be built from it.
|
||||||
- Mobile: NDC 70, 72, 73, 76, 79 — N(S)N exactly 9 digits, i.e. 07X-XXX XX XX (10 digits with trunk 0). Others: 71 mobile broadband/M2M (13 digits), 74 paging, 75 personal numbering, 77 shared cost, 20 freephone, 10 location-independent (10 AXX XX XX, A=1–8), 378 M2M fixed (10 digits).
|
- Mobile: NDC 70, 72, 73, 76, 79 — N(S)N exactly 9 digits, i.e. 07X-XXX XX XX (10 digits with trunk 0). Others: 71 mobile broadband/M2M (13 digits), 74 paging, 75 personal numbering, 77 shared cost, 20 freephone, 10 location-independent (10 AXX XX XX, A=1–8), 378 M2M fixed (10 digits).
|
||||||
- Formatting (Språkrådet/Isof frågelådan, via search excerpts): hyphen after area code, subscriber digits grouped 2–3 from the left: `08-668 01 50`, `031-123 45 67`, `0480-123 45`, `070-123 45 67` (also `0701-23 45 67` accepted). International: `+46 70 123 45 67` (drop trunk 0).
|
- Formatting (Språkrådet/Isof frågelådan, via search excerpts): hyphen after area code, subscriber digits grouped 2–3 from the left: `08-668 01 50`, `031-123 45 67`, `0480-123 45`, `070-123 45 67` (also `0701-23 45 67` accepted). International: `+46 70 123 45 67` (drop trunk 0).
|
||||||
|
|||||||
@@ -1,461 +0,0 @@
|
|||||||
package fejkdata
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// drawSet is one render's reference draws: the unnamed draw group's, and each named one's.
|
|
||||||
type drawSet struct {
|
|
||||||
unnamed draws
|
|
||||||
named map[string]*draws
|
|
||||||
}
|
|
||||||
|
|
||||||
// drawScope is where a render reads its reference paths: a draw set, in the draw group of the
|
|
||||||
// template rendering, and the row a table's format is rendering, which its columns read.
|
|
||||||
type drawScope struct {
|
|
||||||
set *drawSet
|
|
||||||
group string
|
|
||||||
t *table
|
|
||||||
row int
|
|
||||||
}
|
|
||||||
|
|
||||||
// newDrawSet makes the unnamed group's maps where the set is declared, keeping them on that frame's
|
|
||||||
// stack for a render that reads through them; a zero drawSet makes them on its first read instead.
|
|
||||||
func newDrawSet(s *session) drawSet {
|
|
||||||
return drawSet{unnamed: draws{variant: map[string]node{}, value: map[string]draw{}, s: s}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// renderOnce renders n as one render, over draws of its own.
|
|
||||||
func renderOnce(s *session, n node) string {
|
|
||||||
set := drawSet{unnamed: draws{s: s}}
|
|
||||||
return render(s, n, drawScope{set: &set})
|
|
||||||
}
|
|
||||||
|
|
||||||
// in is the scope t renders in: its draw group where it names one, else its caller's.
|
|
||||||
func (sc drawScope) in(t *template) drawScope {
|
|
||||||
if t.drawGroupKey != "" {
|
|
||||||
sc.group = t.drawGroupKey
|
|
||||||
}
|
|
||||||
return sc
|
|
||||||
}
|
|
||||||
|
|
||||||
// draws is the set's draws for sc's draw group. The session is passed in rather than
|
|
||||||
// read out of the set: copying it from there would leak the set's maps to the heap.
|
|
||||||
func (sc drawScope) draws(s *session) *draws {
|
|
||||||
if sc.group == "" {
|
|
||||||
return &sc.set.unnamed
|
|
||||||
}
|
|
||||||
d, drew := sc.set.named[sc.group]
|
|
||||||
if !drew {
|
|
||||||
if sc.set.named == nil {
|
|
||||||
sc.set.named = map[string]*draws{}
|
|
||||||
}
|
|
||||||
d = &draws{variant: map[string]node{}, value: map[string]draw{}, s: s}
|
|
||||||
sc.set.named[strings.Clone(sc.group)] = d // a key from sc would leak sc, and with it every render's draw set, to the heap
|
|
||||||
}
|
|
||||||
return d
|
|
||||||
}
|
|
||||||
|
|
||||||
// drawGroupOf reads a template's "drawGroup" (default ""), which a repeat cannot carry: each
|
|
||||||
// iteration renders in no draw group.
|
|
||||||
func drawGroupOf(m map[string]any, repeat int) (string, error) {
|
|
||||||
v, ok := m["drawGroup"]
|
|
||||||
if !ok {
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
name, ok := v.(string)
|
|
||||||
switch {
|
|
||||||
case !ok:
|
|
||||||
return "", fmt.Errorf("drawGroup must be a string, got %T", v)
|
|
||||||
case name == "":
|
|
||||||
return "", fmt.Errorf(`drawGroup "" is the default, so it has no effect; drop it`)
|
|
||||||
case repeat > 1:
|
|
||||||
return "", fmt.Errorf("drawGroup %q on a repeat names nothing, since each iteration is a render of its own; drop it", name)
|
|
||||||
}
|
|
||||||
return name, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// keyDrawGroup keys t's draw group by the category t sits in, "" for an inline template, so a name
|
|
||||||
// is local to its category.
|
|
||||||
func (t *template) keyDrawGroup(category string) {
|
|
||||||
if t.drawGroup != "" {
|
|
||||||
t.drawGroupKey = category + "/" + t.drawGroup
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkNestedDrawGroup refuses a template beneath one drawing in group that names group again,
|
|
||||||
// short of a repeat or another draw group.
|
|
||||||
func checkNestedDrawGroup(fields map[string]node, group string) error {
|
|
||||||
if group == "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var walk func(path string, n node) error
|
|
||||||
walk = func(path string, n node) error {
|
|
||||||
t, isTemplate := n.(*template)
|
|
||||||
switch {
|
|
||||||
case isTemplate && t.drawGroup == group:
|
|
||||||
return fmt.Errorf("%q names drawGroup %q, the draw group this template draws in already; drop it", path, group)
|
|
||||||
case isTemplate && (t.drawGroup != "" || t.repeat > 1):
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
for _, c := range contained(n) {
|
|
||||||
if err := walk(join(path, c.name), c.node); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
for _, name := range sortedNames(fields) {
|
|
||||||
if err := walk(name, fields[name]); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// drawCheck fences each template of a scope as a render of its own, remembering which nodes read a
|
|
||||||
// reference path.
|
|
||||||
type drawCheck struct {
|
|
||||||
memo map[readsMemo]bool
|
|
||||||
}
|
|
||||||
|
|
||||||
type readsMemo struct {
|
|
||||||
n node
|
|
||||||
stopAtGroup bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkDrawGroup refuses a draw group that splits nothing: one whose render reads every reference
|
|
||||||
// path inside a repeat or a nested draw group, which draw apart from it whatever it names.
|
|
||||||
func (c *drawCheck) checkDrawGroup(path string, n node) error {
|
|
||||||
if t, ok := n.(*template); ok && t.drawGroup != "" && !c.splitsDraws(t) {
|
|
||||||
return fmt.Errorf("%s: drawGroup %q splits nothing, since nothing it renders reads a reference path outside a repeat or a nested drawGroup; drop it", path, t.drawGroup)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *drawCheck) checkDraws(path string, n node) error {
|
|
||||||
t, ok := n.(*template)
|
|
||||||
if !ok {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err := checkOwnFamily(t); err != nil {
|
|
||||||
return fmt.Errorf("%s: %w", path, err)
|
|
||||||
}
|
|
||||||
if !c.readsPath(t) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
w := newDrawWalk()
|
|
||||||
for _, e := range renderEdges(t) {
|
|
||||||
w.edge(t, e, drawAt{group: t.drawGroupKey, route: drawRoute{e.reached(), e.label}})
|
|
||||||
}
|
|
||||||
if err := w.check(); err != nil {
|
|
||||||
return fmt.Errorf("%s: %w", path, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkOwnFamily refuses a table's format or cell that reads, however many templates
|
|
||||||
// away and through a repeat or a draw group too, a table of its own family: a row
|
|
||||||
// rendered whole draws its row without pinning it, so the family would draw apart
|
|
||||||
// from the row being rendered, whichever draws the reaching template holds.
|
|
||||||
func checkOwnFamily(t *template) error {
|
|
||||||
own := t.table
|
|
||||||
if own == nil {
|
|
||||||
own = t.cellOf
|
|
||||||
}
|
|
||||||
if own == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
seen := map[node]bool{}
|
|
||||||
var find func(n node) (renderEdge, *table, bool)
|
|
||||||
find = func(n node) (renderEdge, *table, bool) {
|
|
||||||
if seen[n] {
|
|
||||||
return renderEdge{}, nil, false
|
|
||||||
}
|
|
||||||
seen[n] = true
|
|
||||||
for _, e := range renderEdges(n) {
|
|
||||||
if a, isRef := refRead(n, e.label); isRef {
|
|
||||||
if head, isTable := n.(*template).fields[a.key].(*table); isTable && head.family() == own.family() {
|
|
||||||
return e, head, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if e, head, found := find(e.to); found {
|
|
||||||
return e, head, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return renderEdge{}, nil, false
|
|
||||||
}
|
|
||||||
if e, head, found := find(t); found {
|
|
||||||
return fmt.Errorf("%s reads %s, a table of its own family, which a row of %s rendered whole would draw apart from; read the family from a template beside it, or add the value as a column", e.reached(), head.category, own.category)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkRecordDraws fences the columns of a record — a template compiled at the top without a
|
|
||||||
// repeat — so a load proves the record view of it as well as the string view.
|
|
||||||
func (c *drawCheck) checkRecordDraws(path string, n node) error {
|
|
||||||
t, ok := n.(*template)
|
|
||||||
if !ok || !t.record {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
// A record-only template's format renders nothing, so weigh the columns, not the format.
|
|
||||||
columns := recordColumns(t)
|
|
||||||
reads := false
|
|
||||||
for _, name := range columns {
|
|
||||||
reads = reads || c.readsPath(t.fields[name])
|
|
||||||
}
|
|
||||||
if !reads {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err := checkColumnDraws(t, columns); err != nil {
|
|
||||||
return fmt.Errorf("%s: %w", path, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// readsPath reports whether rendering n reads a reference path, short of a repeat, which renders
|
|
||||||
// over draws of its own.
|
|
||||||
func (c *drawCheck) readsPath(n node) bool { return c.reads(n, false) }
|
|
||||||
|
|
||||||
// splitsDraws reports whether rendering n reads a reference path that n's own draw group answers
|
|
||||||
// for: one outside a repeat and outside a nested draw group, which hold their own draws.
|
|
||||||
func (c *drawCheck) splitsDraws(n node) bool { return c.reads(n, true) }
|
|
||||||
|
|
||||||
// reads walks what rendering n renders for a reference path, stopping at a repeat — and at a nested
|
|
||||||
// draw group when stopAtGroup — since each holds draws of its own.
|
|
||||||
func (c *drawCheck) reads(n node, stopAtGroup bool) bool {
|
|
||||||
k := readsMemo{n, stopAtGroup}
|
|
||||||
if r, done := c.memo[k]; done {
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
r := false
|
|
||||||
for _, e := range renderEdges(n) {
|
|
||||||
if a, isRef := refRead(n, e.label); (isRef && (len(a.tail) > 0 || readsTable(n, a))) || (!repeats(e.to) && !(stopAtGroup && grouped(e.to)) && c.reads(e.to, stopAtGroup)) {
|
|
||||||
r = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if c.memo == nil {
|
|
||||||
c.memo = map[readsMemo]bool{}
|
|
||||||
}
|
|
||||||
c.memo[k] = r
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
|
|
||||||
func repeats(n node) bool {
|
|
||||||
t, isTemplate := n.(*template)
|
|
||||||
return isTemplate && t.repeat > 1
|
|
||||||
}
|
|
||||||
|
|
||||||
func grouped(n node) bool {
|
|
||||||
t, isTemplate := n.(*template)
|
|
||||||
return isTemplate && t.drawGroupKey != ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// readsTable reports whether a reference of n names a table, whose draw the render's
|
|
||||||
// group answers for even when read whole.
|
|
||||||
func readsTable(n node, a arm) bool {
|
|
||||||
t, isTemplate := n.(*template)
|
|
||||||
if !isTemplate {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
_, isTable := t.fields[a.key].(*table)
|
|
||||||
return isTable
|
|
||||||
}
|
|
||||||
|
|
||||||
// refRead is the reference an edge of n reads; false when the edge reads none.
|
|
||||||
func refRead(n node, label string) (arm, bool) {
|
|
||||||
t, isTemplate := n.(*template)
|
|
||||||
if !isTemplate {
|
|
||||||
return arm{}, false
|
|
||||||
}
|
|
||||||
a := splitArm(label, t.refs)
|
|
||||||
return a, isRef(a.key)
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkColumnDraws fences a record's columns as one render.
|
|
||||||
func checkColumnDraws(t *template, columns []string) error {
|
|
||||||
w := newDrawWalk()
|
|
||||||
for _, name := range columns {
|
|
||||||
w.walk(t.fields[name], drawAt{group: t.drawGroupKey, route: drawRoute{spelling: fmt.Sprintf("column %q", name)}})
|
|
||||||
}
|
|
||||||
return w.check()
|
|
||||||
}
|
|
||||||
|
|
||||||
// drawWalk gathers the references one render reads, by draw group, for check to compare.
|
|
||||||
type drawWalk struct {
|
|
||||||
reads []pathRead
|
|
||||||
read map[drawKey]bool
|
|
||||||
seen map[drawVisit]bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// drawAt is where a walk stands: the draw group it draws in, how the render's root reached it, and
|
|
||||||
// the table rows it entered.
|
|
||||||
type drawAt struct {
|
|
||||||
group string
|
|
||||||
route drawRoute
|
|
||||||
alt rowSet
|
|
||||||
}
|
|
||||||
|
|
||||||
// rowSet is the rows a walk entered, one per table: only one row of a table renders, so reads in
|
|
||||||
// two rows of one table never meet, while reads in one row, across its columns and whatever they
|
|
||||||
// reach, do.
|
|
||||||
type rowSet []tablePin
|
|
||||||
|
|
||||||
func (s rowSet) rowOf(t *table) (int, bool) {
|
|
||||||
for _, p := range s {
|
|
||||||
if p.t == t {
|
|
||||||
return p.row, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
|
|
||||||
// enter is s with row r of t, where t is not in it yet; the set is copied, since walks branch.
|
|
||||||
func (s rowSet) enter(t *table, r int) rowSet {
|
|
||||||
if _, in := s.rowOf(t); in {
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
out := append(append(make(rowSet, 0, len(s)+1), s...), tablePin{t, r})
|
|
||||||
sort.Slice(out, func(i, j int) bool { return out[i].t.category < out[j].t.category })
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// key spells the set for a map, by the tables' identities.
|
|
||||||
func (s rowSet) key() string {
|
|
||||||
var b strings.Builder
|
|
||||||
for _, p := range s {
|
|
||||||
fmt.Fprintf(&b, "%p[%d]", p.t, p.row)
|
|
||||||
}
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// alternatives reports whether two reads sit in different rows of one table.
|
|
||||||
func alternatives(a, b drawAt) bool {
|
|
||||||
for _, p := range a.alt {
|
|
||||||
if r, in := b.alt.rowOf(p.t); in && r != p.row {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// drawRoute is how a render reaches a draw: as its author spells it, and the root edge's label.
|
|
||||||
type drawRoute struct{ spelling, label string }
|
|
||||||
|
|
||||||
// pathRead is one reference a render reads: a path, or a bare reference with no tail.
|
|
||||||
type pathRead struct {
|
|
||||||
at drawAt
|
|
||||||
a arm
|
|
||||||
tr *tableRead // set where the reference names a table
|
|
||||||
}
|
|
||||||
|
|
||||||
type drawVisit struct {
|
|
||||||
n node
|
|
||||||
group string
|
|
||||||
alt string
|
|
||||||
}
|
|
||||||
|
|
||||||
type drawKey struct {
|
|
||||||
group string
|
|
||||||
path string
|
|
||||||
alt string
|
|
||||||
}
|
|
||||||
|
|
||||||
func newDrawWalk() *drawWalk {
|
|
||||||
return &drawWalk{read: map[drawKey]bool{}, seen: map[drawVisit]bool{}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// walk follows what rendering n renders. A repeat renders over draws of its own, so the walk stops
|
|
||||||
// there.
|
|
||||||
func (w *drawWalk) walk(n node, at drawAt) {
|
|
||||||
v := drawVisit{n, at.group, at.alt.key()}
|
|
||||||
if w.seen[v] {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.seen[v] = true
|
|
||||||
if repeats(n) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if t, isTemplate := n.(*template); isTemplate && t.drawGroupKey != "" {
|
|
||||||
at.group = t.drawGroupKey
|
|
||||||
}
|
|
||||||
for _, e := range renderEdges(n) {
|
|
||||||
if cell, isCell := e.to.(*template); isCell && cell.cellOf != nil {
|
|
||||||
if at.alt.excludes(cell) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
w.edge(n, e, drawAt{at.group, at.route, at.alt.enter(cell.cellOf, cell.cellRow)})
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
w.edge(n, e, at)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// excludes reports whether a cell's row cannot render with the rows entered: another
|
|
||||||
// row of its table, or a row outside an entered ancestor's.
|
|
||||||
func (s rowSet) excludes(cell *template) bool {
|
|
||||||
for _, p := range s {
|
|
||||||
if p.t == cell.cellOf {
|
|
||||||
return p.row != cell.cellRow
|
|
||||||
}
|
|
||||||
if cell.cellOf.descends(p.t) && !cell.cellOf.under(cell.cellRow, p.t, p.row) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// edge records the reference an edge reads, then walks on with every row the read
|
|
||||||
// pins entered, so a selected row renders only its own cells.
|
|
||||||
func (w *drawWalk) edge(from node, e renderEdge, at drawAt) {
|
|
||||||
if a, reads := refRead(from, e.label); reads {
|
|
||||||
tr := tableReadOf(from.(*template).fields[a.key], a, e.to)
|
|
||||||
if k := (drawKey{at.group, a.path, at.alt.key()}); !w.read[k] {
|
|
||||||
w.read[k] = true
|
|
||||||
w.reads = append(w.reads, pathRead{at, a, tr})
|
|
||||||
}
|
|
||||||
if tr != nil {
|
|
||||||
tr.pins.each(func(t *table, r int) { at.alt = at.alt.enter(t, r) })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
w.walk(e.to, at)
|
|
||||||
}
|
|
||||||
|
|
||||||
// check refuses what one draw per reference path cannot answer for: a read of a level beside a path
|
|
||||||
// another read takes into it, and two reads of one table family that select different rows. Reads
|
|
||||||
// are compared in path order, so which pair is reported does not vary.
|
|
||||||
func (w *drawWalk) check() error {
|
|
||||||
sort.SliceStable(w.reads, func(i, j int) bool {
|
|
||||||
if w.reads[i].at.group != w.reads[j].at.group {
|
|
||||||
return w.reads[i].at.group < w.reads[j].at.group
|
|
||||||
}
|
|
||||||
return w.reads[i].a.path < w.reads[j].a.path
|
|
||||||
})
|
|
||||||
for i, level := range w.reads {
|
|
||||||
for _, into := range w.reads[i+1:] {
|
|
||||||
if into.at.group != level.at.group || alternatives(level.at, into.at) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if strings.HasPrefix(into.a.path, level.a.path+".") && !(level.tr != nil && level.tr.whole) {
|
|
||||||
return overlapError(level.at.route, level.a.name, into)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return checkFamilies(w.reads)
|
|
||||||
}
|
|
||||||
|
|
||||||
func overlapError(route drawRoute, ref string, into pathRead) error {
|
|
||||||
return fmt.Errorf("%s renders a level that %s reads a path into; name the fields you want instead, or draw them apart with a drawGroup", route.spelled(ref), into.at.route.spelled(into.a.name))
|
|
||||||
}
|
|
||||||
|
|
||||||
// spelled names the route, and the reference it reaches a draw by where its root edge is not that
|
|
||||||
// reference.
|
|
||||||
func (r drawRoute) spelled(ref string) string {
|
|
||||||
if ref == "" || ref == r.label {
|
|
||||||
return r.spelling
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s with {%s}", r.spelling, ref)
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
package fejkdata
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/larvit/fejkdata/internal/drawstate"
|
||||||
|
"github.com/larvit/fejkdata/internal/invariant"
|
||||||
|
)
|
||||||
|
|
||||||
|
// readValue is what one read drew: its text, and whether it landed on a null.
|
||||||
|
type readValue struct {
|
||||||
|
text string
|
||||||
|
null bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// frameStack is the frames of the name scopes rendering, innermost last. A render's env points
|
||||||
|
// at it, and a repeat iteration's shares it: holding the frames in a renderEnv would move every
|
||||||
|
// render's env to the heap.
|
||||||
|
type frameStack struct {
|
||||||
|
frames []*pickFrame
|
||||||
|
}
|
||||||
|
|
||||||
|
func (st *frameStack) push(f *pickFrame) int {
|
||||||
|
mark := len(st.frames)
|
||||||
|
st.frames = append(st.frames, f)
|
||||||
|
return mark
|
||||||
|
}
|
||||||
|
|
||||||
|
func (st *frameStack) pop(mark int) { st.frames = st.frames[:mark] }
|
||||||
|
|
||||||
|
// renderEnv is where a render reads: the frames of the name scopes rendering; and row, the row
|
||||||
|
// of the table rendering, which its columns read. It passes by value, and what is read from it
|
||||||
|
// reaches a map key, an interface or a func value only as a copy; else env, and with it every
|
||||||
|
// render's frames, moves to the heap.
|
||||||
|
type renderEnv struct {
|
||||||
|
frames *frameStack
|
||||||
|
row renderedRow
|
||||||
|
// base is the depth of the frame stack when the read rendering the category entered it,
|
||||||
|
// below which no frame is the category's.
|
||||||
|
base int
|
||||||
|
// pick is the named pick being rendered, nil outside one; pickAt is the key in it of what renders.
|
||||||
|
pick *namedPick
|
||||||
|
pickAt pickKey
|
||||||
|
}
|
||||||
|
|
||||||
|
type renderedRow struct {
|
||||||
|
t *table
|
||||||
|
index int
|
||||||
|
}
|
||||||
|
|
||||||
|
// at returns the env n renders in, n being the leaf of a path whose rows sit in pins; where n
|
||||||
|
// is a table's row or column, the env carries the row pins holds for that table.
|
||||||
|
func (env renderEnv) at(n node, pins *pinSet) renderEnv {
|
||||||
|
switch n := n.(type) {
|
||||||
|
case *tableRow:
|
||||||
|
env.row = renderedRow{n.t, pins.MustRow(n.t.rows)}
|
||||||
|
case *tableColumn:
|
||||||
|
env.row = renderedRow{n.t, pins.MustRow(n.t.rows)}
|
||||||
|
}
|
||||||
|
return env
|
||||||
|
}
|
||||||
|
|
||||||
|
func (env renderEnv) rowOf(t *table) int {
|
||||||
|
if env.row.t != t {
|
||||||
|
panic(invariant.Broken("a column of %s renders in an env holding no row of it", t.rows.Segment()))
|
||||||
|
}
|
||||||
|
return env.row.index
|
||||||
|
}
|
||||||
|
|
||||||
|
// enter starts a read landing on n: the read sees no frame opened before it, and gets a frame for
|
||||||
|
// each name scope around n, from memo where there is one, so reads sharing memo read one pick of
|
||||||
|
// each name. It returns the mark that closes those frames.
|
||||||
|
func (env renderEnv) enter(n node, memo *drawMemo) (renderEnv, int) {
|
||||||
|
env = env.entering()
|
||||||
|
for scope := scopeAround(n); scope != nil; scope = scope.up {
|
||||||
|
if len(scope.order) > 0 {
|
||||||
|
env.frames.push(memo.enteredFrame(scope))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return env, env.base
|
||||||
|
}
|
||||||
|
|
||||||
|
// entering is env as a read entering a category sees it: no frame opened before it, and no pick.
|
||||||
|
func (env renderEnv) entering() renderEnv {
|
||||||
|
env.base, env.pick = len(env.frames.frames), nil
|
||||||
|
return env
|
||||||
|
}
|
||||||
|
|
||||||
|
// renderFrame opens a fresh frame of t's scope, where t binds names and no read entering the
|
||||||
|
// category opened one, returning the mark that closes it, or -1.
|
||||||
|
func (env renderEnv) renderFrame(t *template) int {
|
||||||
|
own := t.ownScope()
|
||||||
|
if own == nil || env.frameOf(own) != nil {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
return env.frames.push(newPickFrame(own))
|
||||||
|
}
|
||||||
|
|
||||||
|
// frameOf is the frame of scope rendering since the read entering the category, nil where none is.
|
||||||
|
func (env renderEnv) frameOf(scope *nameScope) *pickFrame {
|
||||||
|
for i := len(env.frames.frames) - 1; i >= env.base; i-- {
|
||||||
|
if f := env.frames.frames[i]; f.scope == scope {
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// drawRowOf draws the row a render of t reads: inside the pick's rows where t renders as part of one.
|
||||||
|
func (env renderEnv) drawRowOf(s *drawstate.State, t *table) int {
|
||||||
|
if env.pick != nil {
|
||||||
|
return t.rows.DrawIn(s, &env.pick.pins)
|
||||||
|
}
|
||||||
|
return t.rows.Draw(s)
|
||||||
|
}
|
||||||
@@ -1,319 +0,0 @@
|
|||||||
package fejkdata
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// tablePin is one table's pinned row.
|
|
||||||
type tablePin struct {
|
|
||||||
t *table
|
|
||||||
row int
|
|
||||||
}
|
|
||||||
|
|
||||||
// pinned is the row the render pinned for t, if any.
|
|
||||||
func (d *draws) pinned(t *table) (int, bool) {
|
|
||||||
for _, p := range d.pins[:d.npins] {
|
|
||||||
if p.t == t {
|
|
||||||
return p.row, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
r, ok := d.more[t]
|
|
||||||
return r, ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// mustRow is the row pinned for t, which the walk reaching a column pinned.
|
|
||||||
func (d *draws) mustRow(t *table) int {
|
|
||||||
r, ok := d.pinned(t)
|
|
||||||
if !ok {
|
|
||||||
panic(fmt.Sprintf("fejkdata: a column of %s is rendered with no row pinned", t.category))
|
|
||||||
}
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
|
|
||||||
// pin pins row r of t, and the rows of t's ancestors it links to.
|
|
||||||
func (d *draws) pin(t *table, r int) {
|
|
||||||
for {
|
|
||||||
if _, done := d.pinned(t); done {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if d.npins < len(d.pins) {
|
|
||||||
d.pins[d.npins] = tablePin{t, r}
|
|
||||||
d.npins++
|
|
||||||
} else {
|
|
||||||
if d.more == nil {
|
|
||||||
d.more = map[*table]int{}
|
|
||||||
}
|
|
||||||
d.more[t] = r
|
|
||||||
}
|
|
||||||
if t.parentT == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
t, r = t.parentT, t.parentRow(r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// each calls fn for every pinned row, in pin order, the spilled ones by table name.
|
|
||||||
func (d *draws) each(fn func(t *table, r int)) {
|
|
||||||
for _, p := range d.pins[:d.npins] {
|
|
||||||
fn(p.t, p.row)
|
|
||||||
}
|
|
||||||
spilled := make([]*table, 0, len(d.more))
|
|
||||||
for t := range d.more {
|
|
||||||
spilled = append(spilled, t)
|
|
||||||
}
|
|
||||||
sort.Slice(spilled, func(i, j int) bool { return spilled[i].category < spilled[j].category })
|
|
||||||
for _, t := range spilled {
|
|
||||||
fn(t, d.more[t])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// rowOf is the render's row of t: the one pinned, else one drawn inside the
|
|
||||||
// nearest pinned ancestor — its parent drawn inside that first where the ancestor
|
|
||||||
// is further up — or over the whole table, and pinned with its ancestors.
|
|
||||||
func (d *draws) rowOf(t *table) int {
|
|
||||||
if r, ok := d.pinned(t); ok {
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
r := -1
|
|
||||||
for a := t.parentT; a != nil && r < 0; a = a.parentT {
|
|
||||||
if _, ok := d.pinned(a); !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if t.parentT != a {
|
|
||||||
d.rowOf(t.parentT)
|
|
||||||
}
|
|
||||||
pr, _ := d.pinned(t.parentT)
|
|
||||||
r = t.drawUnder(d.s, pr)
|
|
||||||
}
|
|
||||||
if r < 0 {
|
|
||||||
r = t.draw(d.s)
|
|
||||||
}
|
|
||||||
d.pin(t, r)
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
|
|
||||||
// pinRow pins row r of t where it agrees with the rows pinned before it.
|
|
||||||
func (d *draws) pinRow(t *table, r int) error {
|
|
||||||
if pr, ok := d.pinned(t); ok && pr != r {
|
|
||||||
return fmt.Errorf("%s and %s are two rows of %s", t.selectorSpelling(pr), t.selectorSpelling(r), t.path)
|
|
||||||
}
|
|
||||||
for a := t.parentT; a != nil; a = a.parentT {
|
|
||||||
if pa, ok := d.pinned(a); ok && !t.under(r, a, pa) {
|
|
||||||
return fmt.Errorf("%s is not inside %s", t.selectorSpelling(r), a.selectorSpelling(pa))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
d.pin(t, r)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// selectRow pins the row a selector names.
|
|
||||||
func (d *draws) selectRow(t *table, sel string) error {
|
|
||||||
r, err := t.find(sel, d)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return d.pinRow(t, r)
|
|
||||||
}
|
|
||||||
|
|
||||||
// inside keeps the rows of t that sit inside every pinned ancestor.
|
|
||||||
func (d *draws) inside(t *table, rows []int) []int {
|
|
||||||
for a := t.parentT; a != nil; a = a.parentT {
|
|
||||||
pa, ok := d.pinned(a)
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var kept []int
|
|
||||||
for _, r := range rows {
|
|
||||||
if t.under(r, a, pa) {
|
|
||||||
kept = append(kept, r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
rows = kept
|
|
||||||
}
|
|
||||||
return rows
|
|
||||||
}
|
|
||||||
|
|
||||||
// tableRead is what a reference path reads of a table family: the table its head
|
|
||||||
// names, the rows its selectors pin, the tables it draws — those it walks with no
|
|
||||||
// row pinned, and their unpinned ancestors — each selector's spelling, and whether
|
|
||||||
// it lands on a row rendered whole.
|
|
||||||
type tableRead struct {
|
|
||||||
head *table
|
|
||||||
pins draws
|
|
||||||
drawn map[*table]bool
|
|
||||||
sels []tableSel
|
|
||||||
whole bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// tableSel is one selector on the way: the table it selects a row of, and the path
|
|
||||||
// as written up to and including it.
|
|
||||||
type tableSel struct {
|
|
||||||
t *table
|
|
||||||
spelling string
|
|
||||||
}
|
|
||||||
|
|
||||||
// tableReadOf replays a reference's selectors through the walk a render uses, so
|
|
||||||
// two paths pinning one row by different routes compare equal. checkPath proved
|
|
||||||
// each selector names a row.
|
|
||||||
func tableReadOf(head node, a arm, leaf node) *tableRead {
|
|
||||||
t, isTable := head.(*table)
|
|
||||||
if !isTable {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
tr := &tableRead{head: t, drawn: map[*table]bool{}}
|
|
||||||
_, _ = walkPath(t, a.tail, pathWalk{pins: &tr.pins})
|
|
||||||
written := a.name[:len(a.name)-len(joinSegments(a.tail))]
|
|
||||||
cur := t
|
|
||||||
tr.draws(cur)
|
|
||||||
for i, seg := range a.tail {
|
|
||||||
switch d := cur.descendant(seg); {
|
|
||||||
case isSelector(seg):
|
|
||||||
tr.sels = append(tr.sels, tableSel{cur, written + joinSegments(a.tail[:i+1])})
|
|
||||||
case d != nil:
|
|
||||||
cur = d
|
|
||||||
tr.draws(cur)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_, tr.whole = leaf.(*row)
|
|
||||||
return tr
|
|
||||||
}
|
|
||||||
|
|
||||||
// draws marks t and its ancestors drawn, up to the first the read pins.
|
|
||||||
func (r *tableRead) draws(t *table) {
|
|
||||||
for ; t != nil; t = t.parentT {
|
|
||||||
if _, pinned := r.pins.pinned(t); pinned {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
r.drawn[t] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// joinSegments spells segments as a path: a selector attaches to the name before it.
|
|
||||||
func joinSegments(segs []string) string {
|
|
||||||
var b strings.Builder
|
|
||||||
for _, s := range segs {
|
|
||||||
if b.Len() > 0 && !isSelector(s) {
|
|
||||||
b.WriteByte('.')
|
|
||||||
}
|
|
||||||
b.WriteString(s)
|
|
||||||
}
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// family is the table a chain of parents ends at.
|
|
||||||
func (t *table) family() *table {
|
|
||||||
for t.parentT != nil {
|
|
||||||
t = t.parentT
|
|
||||||
}
|
|
||||||
return t
|
|
||||||
}
|
|
||||||
|
|
||||||
// selected is the selector in r on t, or on the nearest ancestor of t it selects.
|
|
||||||
func (r *tableRead) selected(t *table) (tableSel, bool) {
|
|
||||||
for ; t != nil; t = t.parentT {
|
|
||||||
for _, s := range r.sels {
|
|
||||||
if s.t == t {
|
|
||||||
return s, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return tableSel{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkFamilies refuses reads of one table family in one draw group that cannot
|
|
||||||
// read one consistent draw: a table rendered whole beside a path into the family,
|
|
||||||
// a table one read draws that another pins, and two reads pinning different rows.
|
|
||||||
// The reads come sorted by group and path, so which pair is reported does not vary.
|
|
||||||
func checkFamilies(reads []pathRead) error {
|
|
||||||
for i, r := range reads {
|
|
||||||
if r.tr == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, o := range reads[:i] {
|
|
||||||
if o.tr == nil || o.at.group != r.at.group || alternatives(o.at, r.at) || o.tr.head.family() != r.tr.head.family() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := checkFamilyPair(o, r); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return replayPairs(reads)
|
|
||||||
}
|
|
||||||
|
|
||||||
// replayPairs replays every two reads of one draw group that can render together into
|
|
||||||
// one draws, the earlier read first. A pin conflicts with one earlier pin, never with a
|
|
||||||
// combination, so pairs find every conflict a full replay would.
|
|
||||||
func replayPairs(reads []pathRead) error {
|
|
||||||
for i, r := range reads {
|
|
||||||
if r.tr == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, o := range reads[i+1:] {
|
|
||||||
if o.tr == nil || o.at.group != r.at.group || alternatives(r.at, o.at) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var d draws
|
|
||||||
if err := r.tr.replay(&d); err != nil {
|
|
||||||
return conflict(r, err)
|
|
||||||
}
|
|
||||||
if err := o.tr.replay(&d); err != nil {
|
|
||||||
return conflict(o, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func conflict(r pathRead, err error) error {
|
|
||||||
return fmt.Errorf("%s: %w; select the same rows in every path into the family, or draw them apart with a drawGroup", r.at.route.spelled(r.a.name), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// replay pins the read's rows into d, where they agree with the rows pinned before.
|
|
||||||
func (r *tableRead) replay(d *draws) error {
|
|
||||||
var err error
|
|
||||||
r.pins.each(func(t *table, row int) {
|
|
||||||
if err == nil {
|
|
||||||
err = d.pinRow(t, row)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkFamilyPair refuses a table read whole beside a path into its family, and a
|
|
||||||
// table one read draws that the other pins, since which token renders first would
|
|
||||||
// then decide the row.
|
|
||||||
func checkFamilyPair(a, b pathRead) error {
|
|
||||||
for _, pair := range [][2]pathRead{{a, b}, {b, a}} {
|
|
||||||
x, y := pair[0], pair[1]
|
|
||||||
if len(x.a.tail) == 0 && len(y.a.tail) > 0 {
|
|
||||||
return overlapError(x.at.route, x.a.name, y)
|
|
||||||
}
|
|
||||||
if drawn := x.tr.drawnOf(&y.tr.pins); drawn != nil {
|
|
||||||
if s, ok := y.tr.selected(x.tr.head); ok && len(x.tr.sels) == 0 {
|
|
||||||
tail := x.a.tail
|
|
||||||
if s.t != x.tr.head {
|
|
||||||
tail = append([]string{x.tr.head.category}, tail...)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("%s draws %s, which %s selects a row of; write {%s.%s}, or draw them apart with a drawGroup",
|
|
||||||
x.at.route.spelled(x.a.name), drawn.category, y.at.route.spelled(y.a.name), s.spelling, joinSegments(tail))
|
|
||||||
}
|
|
||||||
return fmt.Errorf("%s draws %s, which %s selects a row of; select that row in both, or draw them apart with a drawGroup",
|
|
||||||
x.at.route.spelled(x.a.name), drawn.category, y.at.route.spelled(y.a.name))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// drawnOf is a table the read draws that pins holds a row of, if any.
|
|
||||||
func (r *tableRead) drawnOf(pins *draws) *table {
|
|
||||||
var found *table
|
|
||||||
pins.each(func(t *table, _ int) {
|
|
||||||
if found == nil && r.drawn[t] {
|
|
||||||
found = t
|
|
||||||
}
|
|
||||||
})
|
|
||||||
return found
|
|
||||||
}
|
|
||||||
+83
-203
@@ -1,81 +1,51 @@
|
|||||||
// Package fejkdata generates fake data from recursive JSON templates.
|
|
||||||
//
|
|
||||||
// Data lives in JSON, not in Go. A generator starts from the shipped data set and
|
|
||||||
// layers any directories you add; folders and files become a dot-path namespace,
|
|
||||||
// then generate values by path:
|
|
||||||
//
|
|
||||||
// f, _ := fejkdata.New(fejkdata.WithSeed(42))
|
|
||||||
// f.Fake("sv_SE.address") // "Järvedsvägen 43\n891 77 Järved"
|
|
||||||
// f.Fake("sv_SE.address.locality") // "Sundbyberg": a second call draws afresh
|
|
||||||
//
|
|
||||||
// Several sources merge in order, the last winning a name clash, so custom data
|
|
||||||
// layers over the built-ins. The JSON template format is documented in the README.
|
|
||||||
package fejkdata
|
package fejkdata
|
||||||
|
|
||||||
import (
|
import (
|
||||||
crand "crypto/rand"
|
|
||||||
"embed"
|
|
||||||
"encoding/binary"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"math/rand/v2"
|
|
||||||
"os"
|
|
||||||
"reflect"
|
"reflect"
|
||||||
"sort"
|
"sort"
|
||||||
"sync"
|
"sync"
|
||||||
)
|
|
||||||
|
|
||||||
//go:embed data
|
"github.com/larvit/fejkdata/internal/datafiles"
|
||||||
var shippedFS embed.FS
|
"github.com/larvit/fejkdata/internal/drawstate"
|
||||||
|
)
|
||||||
|
|
||||||
// MaxRepeat caps a repeat, and caps the renders nested repeats multiply to along
|
// MaxRepeat caps a repeat, and caps the renders nested repeats multiply to along
|
||||||
// any one path; the CLI's --repeat shares it.
|
// any one path; the CLI's --repeat shares it.
|
||||||
const MaxRepeat = 1 << 20
|
const MaxRepeat = 1 << 20
|
||||||
|
|
||||||
var _ [^uint(0)>>63 - 1]struct{} // 64-bit only, per the README's Decisions
|
// Fails to compile where uint is narrower than 64 bits: docs/decisions.md#64-bit-targets-only
|
||||||
|
var _ [^uint(0)>>63 - 1]struct{}
|
||||||
|
|
||||||
// ErrNoData is returned by New when no source is loaded at all.
|
// Generator generates fake data from a namespace tree. Create one with [New].
|
||||||
var ErrNoData = errors.New("no data: WithoutShippedData needs at least one WithDataPath or WithDataFS")
|
// It is safe for concurrent use; a sequence is reproducible only when drawn
|
||||||
|
// from one goroutine.
|
||||||
// Generator generates fake data from a loaded namespace tree. Create one with [New].
|
|
||||||
// It is safe for concurrent use; a seeded sequence is reproducible only when drawn
|
|
||||||
// from one goroutine. The compiled tree is immutable after [New], so List reads it
|
|
||||||
// without a lock; mu guards the rest, which every other entry point takes.
|
|
||||||
type Generator struct {
|
type Generator struct {
|
||||||
mu sync.Mutex
|
// mu guards drawState, structs and root, which gains an indexed category on the
|
||||||
rand *session
|
// first call reaching it.
|
||||||
categories map[string]node
|
mu sync.Mutex
|
||||||
root folder // the categories as the node a path walks from, owned here so a walk allocates none
|
drawState *drawstate.State
|
||||||
set drawSet // one Fake's draws, owned here so a walk pinning rows keeps them off the heap
|
root folder // the categories as the node a path walks from, owned here so a walk allocates none
|
||||||
records map[node]recordShape
|
structs map[reflect.Type]structResult
|
||||||
structs map[reflect.Type]structResult
|
|
||||||
}
|
|
||||||
|
|
||||||
// session is one generator's mutable render state: the seeded rng plus the {seq()}
|
|
||||||
// counters.
|
|
||||||
type session struct {
|
|
||||||
*rand.Rand
|
|
||||||
counters map[string]uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *session) next(key string) uint64 {
|
|
||||||
s.counters[key]++
|
|
||||||
return s.counters[key]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type config struct {
|
type config struct {
|
||||||
seed uint64
|
seed uint64
|
||||||
seeded bool
|
seeded bool
|
||||||
shipped bool
|
sources []datafiles.Source
|
||||||
sources []dataSource
|
nilModules error
|
||||||
|
// dataGiven is whether a data option was passed, even WithDataFS(), which loads none.
|
||||||
|
dataGiven bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Option configures a [Generator].
|
// Option configures a [Generator].
|
||||||
type Option func(*config)
|
type Option func(*config)
|
||||||
|
|
||||||
// WithSeed makes output reproducible: two generators with the same seed and data
|
// WithSeed seeds the generator, as [New] requires: two generators with the same seed,
|
||||||
// emit identical sequences.
|
// version and data emit identical sequences. For a new seed each run, set
|
||||||
|
// seed := rand.Uint64() (math/rand/v2), and log it to replay the run.
|
||||||
func WithSeed(seed uint64) Option {
|
func WithSeed(seed uint64) Option {
|
||||||
return func(c *config) { c.seed, c.seeded = seed, true }
|
return func(c *config) { c.seed, c.seeded = seed, true }
|
||||||
}
|
}
|
||||||
@@ -84,175 +54,85 @@ func WithSeed(seed uint64) Option {
|
|||||||
// layer several; the last wins a name clash.
|
// layer several; the last wins a name clash.
|
||||||
func WithDataPath(dir string) Option {
|
func WithDataPath(dir string) Option {
|
||||||
return func(c *config) {
|
return func(c *config) {
|
||||||
c.sources = append(c.sources, dataSource{fsys: os.DirFS(dir), label: dir, onDisk: true, path: dir})
|
c.sources, c.dataGiven = append(c.sources, datafiles.Dir(dir)), true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithDataFS layers a data tree held in an [fs.FS], such as an embed.FS of your own.
|
// WithDataFS layers modules, each a data tree held in an [fs.FS], over what is loaded
|
||||||
func WithDataFS(fsys fs.FS) Option {
|
// before them, in order. A module may be a data package's FS, such as one of
|
||||||
return func(c *config) { c.sources = append(c.sources, dataSource{fsys: fsys}) }
|
// github.com/larvit/fejkdata/data's Modules, or an embed.FS of your own. WithDataFS()
|
||||||
|
// loads no data.
|
||||||
|
func WithDataFS(modules ...fs.FS) Option {
|
||||||
|
return func(c *config) {
|
||||||
|
for i, fsys := range modules {
|
||||||
|
if fsys == nil {
|
||||||
|
c.nilModules = errors.Join(c.nilModules, fmt.Errorf("fejkdata: WithDataFS module %d is nil", i+1))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c.sources = append(c.sources, datafiles.FS(fsys))
|
||||||
|
}
|
||||||
|
c.dataGiven = true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithoutShippedData leaves the shipped data set out, so only the sources given
|
var errNoSeed = errors.New(`fejkdata: no seed: add fejkdata.WithSeed(n), where the same n repeats a run's output; for a new seed each run, set n := rand.Uint64() (import "math/rand/v2") and log n to replay the run`)
|
||||||
// with [WithDataPath] and [WithDataFS] load.
|
|
||||||
func WithoutShippedData() Option {
|
|
||||||
return func(c *config) { c.shipped = false }
|
|
||||||
}
|
|
||||||
|
|
||||||
// New builds a generator from the shipped data set and the options' sources, merged
|
var errNoData = errors.New(`fejkdata: no data: add fejkdata.WithDataFS(data.Modules()...) (import "github.com/larvit/fejkdata/data") for all shipped data, WithDataFS/WithDataPath for your own, or WithDataFS() for none`)
|
||||||
// in order with the last winning a name clash. Each JSON file becomes a category
|
|
||||||
// named after the file (address.json -> "address") and each subdirectory a
|
// ErrLoad marks data that fails to load: in New, or on the first call reaching a category
|
||||||
// namespace segment. It errors on a missing directory, invalid JSON, invalid data,
|
// of an indexed source.
|
||||||
// or no data at all.
|
var ErrLoad = errors.New("data fails to load")
|
||||||
|
|
||||||
|
// loadError is a load's error, unchanged in its text, matching ErrLoad.
|
||||||
|
type loadError struct{ error }
|
||||||
|
|
||||||
|
func (e loadError) Unwrap() error { return e.error }
|
||||||
|
func (loadError) Is(target error) bool { return target == ErrLoad }
|
||||||
|
|
||||||
|
// New builds a generator from the options' sources, merged in order with the last
|
||||||
|
// winning a name clash; it fails where no [WithSeed] gives the seed, or no [WithDataFS]
|
||||||
|
// or [WithDataPath] names the data to load. Each JSON file becomes a category named
|
||||||
|
// after the file (address.json -> "address") and each subdirectory a namespace segment.
|
||||||
|
// It errors on a missing directory, invalid JSON or invalid data. Given only
|
||||||
|
// WithDataFS() with no modules, New loads no data, and the generator renders templates
|
||||||
|
// that read none. A source whose .fejkdata.json carries an index, as each shipped
|
||||||
|
// module's does, loads each category on the first call reaching it. Data failing to
|
||||||
|
// load, in New or on that first call, gives an error matching [ErrLoad].
|
||||||
func New(opts ...Option) (*Generator, error) {
|
func New(opts ...Option) (*Generator, error) {
|
||||||
c := config{shipped: true}
|
var c config
|
||||||
for _, opt := range opts {
|
for _, opt := range opts {
|
||||||
opt(&c)
|
opt(&c)
|
||||||
}
|
}
|
||||||
var sources []dataSource
|
if err := c.misuse(); err != nil {
|
||||||
if c.shipped {
|
return nil, err
|
||||||
sources = append(sources, dataSource{fsys: shippedFS, root: "data"})
|
|
||||||
}
|
}
|
||||||
sources = append(sources, c.sources...)
|
root, err := loadSources(c.sources)
|
||||||
if len(sources) == 0 {
|
|
||||||
return nil, fmt.Errorf("fejkdata: %w", ErrNoData)
|
|
||||||
}
|
|
||||||
cats, err := loadData(sources)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("fejkdata: %w", err)
|
return nil, fmt.Errorf("fejkdata: %w", loadError{err})
|
||||||
}
|
}
|
||||||
rng, err := newRand(c.seed, c.seeded)
|
return &Generator{drawState: drawstate.New(c.seed), root: root}, nil
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("fejkdata: %w", err)
|
|
||||||
}
|
|
||||||
return &Generator{rand: rng, categories: cats}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// List returns the sorted dotted paths Fake can render: every category, the dotted
|
// List returns the sorted dotted paths Fake renders: each category and every field,
|
||||||
// fields within a template, and folder segments. A choice consumes no segment, so a
|
// column and linked table below it, a direct descent at a time, listing a level
|
||||||
// path continues through one only where every variant carries it, which is the
|
// carrying a repeat but nothing below it. A choice consumes no segment, so a path
|
||||||
// rule Fake applies too: List is the set of paths Fake accepts.
|
// continues through one only where every variant carries it.
|
||||||
func (f *Generator) List() []string {
|
func (f *Generator) List() []string {
|
||||||
var out []string
|
f.mu.Lock()
|
||||||
for _, name := range sortedNames(f.categories) {
|
defer f.mu.Unlock()
|
||||||
for _, p := range paths(f.categories[name]) {
|
out := paths(&f.root, false)
|
||||||
out = append(out, join(name, p))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
sort.Strings(out)
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// paths lists the dot paths addressable from n, relative to it, where "" is n
|
// misuse names every mistake in the options, before any data loads.
|
||||||
// itself. A folder has no value of its own, so it contributes only its children's.
|
func (c config) misuse() error {
|
||||||
func paths(n node) []string {
|
var err error
|
||||||
switch n := n.(type) {
|
if !c.dataGiven {
|
||||||
case *folder:
|
err = errNoData
|
||||||
var out []string
|
|
||||||
for _, name := range sortedNames(n.children) {
|
|
||||||
for _, p := range paths(n.children[name]) {
|
|
||||||
out = append(out, join(name, p))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case *template:
|
|
||||||
out := []string{""}
|
|
||||||
for _, name := range sortedNames(n.fields) {
|
|
||||||
if isRef(name) { // a binding, not a path segment
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, p := range paths(n.fields[name]) {
|
|
||||||
out = append(out, join(name, p))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case *null:
|
|
||||||
return []string{""}
|
|
||||||
case *table:
|
|
||||||
return tablePaths(n)
|
|
||||||
case *column, *row:
|
|
||||||
return []string{""}
|
|
||||||
case *choice:
|
|
||||||
out := []string{""}
|
|
||||||
for p := range n.shared {
|
|
||||||
out = append(out, p)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
}
|
||||||
return nil
|
if !c.seeded {
|
||||||
}
|
err = errors.Join(err, errNoSeed)
|
||||||
|
}
|
||||||
// tablePaths is a table's columns, then each table linked to it under its name: the
|
return errors.Join(err, c.nilModules)
|
||||||
// direct descents, a step at a time.
|
|
||||||
func tablePaths(t *table) []string {
|
|
||||||
out := append([]string{""}, t.columns...)
|
|
||||||
sort.Strings(out[1:])
|
|
||||||
children := make([]string, 0, len(t.children))
|
|
||||||
for name := range t.children {
|
|
||||||
children = append(children, name)
|
|
||||||
}
|
|
||||||
sort.Strings(children)
|
|
||||||
for _, name := range children {
|
|
||||||
for _, p := range paths(t.children[name]) {
|
|
||||||
out = append(out, join(name, p))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// sharedPaths is the sub-paths every item carries — the only ones a path may step
|
|
||||||
// through a choice to reach. It intersects, bailing as soon as the set
|
|
||||||
// is empty, which is immediate for a choice of plain strings.
|
|
||||||
func sharedPaths(items []node) map[string]bool {
|
|
||||||
shared := subPaths(items[0])
|
|
||||||
for _, it := range items[1:] {
|
|
||||||
if len(shared) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
next := subPaths(it)
|
|
||||||
for p := range shared {
|
|
||||||
if !next[p] {
|
|
||||||
delete(shared, p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return shared
|
|
||||||
}
|
|
||||||
|
|
||||||
// subPaths is paths(n) as a set, without the empty path that means n itself.
|
|
||||||
func subPaths(n node) map[string]bool {
|
|
||||||
out := map[string]bool{}
|
|
||||||
for _, p := range paths(n) {
|
|
||||||
if p != "" {
|
|
||||||
out[p] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func join(prefix, name string) string {
|
|
||||||
switch {
|
|
||||||
case prefix == "":
|
|
||||||
return name
|
|
||||||
case name == "":
|
|
||||||
return prefix
|
|
||||||
}
|
|
||||||
return prefix + "." + name
|
|
||||||
}
|
|
||||||
|
|
||||||
// randomBytes seeds an unseeded generator.
|
|
||||||
var randomBytes = crand.Read
|
|
||||||
|
|
||||||
func newRand(seed uint64, seeded bool) (*session, error) {
|
|
||||||
var r *rand.Rand
|
|
||||||
if seeded {
|
|
||||||
r = rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15))
|
|
||||||
} else {
|
|
||||||
var b [16]byte
|
|
||||||
if _, err := randomBytes(b[:]); err != nil {
|
|
||||||
return nil, fmt.Errorf("seeding from crypto/rand: %w", err)
|
|
||||||
}
|
|
||||||
r = rand.New(rand.NewPCG(binary.LittleEndian.Uint64(b[:8]), binary.LittleEndian.Uint64(b[8:])))
|
|
||||||
}
|
|
||||||
return &session{Rand: r, counters: map[string]uint64{}}, nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
+73
-14
@@ -3,11 +3,18 @@ package fejkdata
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
|
||||||
|
"github.com/larvit/fejkdata/data"
|
||||||
|
"github.com/larvit/fejkdata/internal/drawstate"
|
||||||
|
"github.com/larvit/fejkdata/internal/jsonvalue"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func withShipped() Option { return WithDataFS(data.Modules()...) }
|
||||||
|
|
||||||
// newGenerator creates a generator over a single data directory, failing on
|
// newGenerator creates a generator over a single data directory, failing on
|
||||||
// error. Most tests load one dir; newGeneratorN loads several (last-loaded wins
|
// error. Most tests load one dir; newGeneratorN loads several (last-loaded wins
|
||||||
// on conflicts).
|
// on conflicts).
|
||||||
@@ -18,17 +25,28 @@ func newGenerator(t *testing.T, dir string, opts ...Option) *Generator {
|
|||||||
|
|
||||||
func newGeneratorN(t *testing.T, dirs []string, opts ...Option) *Generator {
|
func newGeneratorN(t *testing.T, dirs []string, opts ...Option) *Generator {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
all := []Option{WithoutShippedData()}
|
var all []Option
|
||||||
for _, dir := range dirs {
|
for _, dir := range dirs {
|
||||||
all = append(all, WithDataPath(dir))
|
all = append(all, WithDataPath(dir))
|
||||||
}
|
}
|
||||||
f, err := New(append(all, opts...)...)
|
f, err := New(append(append(all, WithSeed(1)), opts...)...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("New(%q): %v", dirs, err)
|
t.Fatalf("New(%q): %v", dirs, err)
|
||||||
}
|
}
|
||||||
return f
|
return f
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// newBareModule loads the shipped module in data/<module> without its manifest, so its
|
||||||
|
// categories sit at the root, failing on error.
|
||||||
|
func newBareModule(t *testing.T, module string, opts ...Option) *Generator {
|
||||||
|
t.Helper()
|
||||||
|
f, err := New(append([]Option{WithDataFS(withoutManifest{os.DirFS(filepath.Join("data", module))}), WithSeed(1)}, opts...)...)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New(%s without its manifest): %v", module, err)
|
||||||
|
}
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
// fake generates a value, failing the test on error.
|
// fake generates a value, failing the test on error.
|
||||||
func fake(t *testing.T, f *Generator, path string) string {
|
func fake(t *testing.T, f *Generator, path string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -40,31 +58,32 @@ func fake(t *testing.T, f *Generator, path string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestNewMissingDirectory(t *testing.T) {
|
func TestNewMissingDirectory(t *testing.T) {
|
||||||
_, err := New(WithoutShippedData(), WithDataPath("data/de_DE"))
|
_, err := New(WithSeed(1), WithDataPath("data/de_DE"))
|
||||||
if err == nil || !strings.Contains(err.Error(), "de_DE") || !errors.Is(err, fs.ErrNotExist) {
|
if err == nil || !strings.Contains(err.Error(), "de_DE") || !errors.Is(err, fs.ErrNotExist) {
|
||||||
t.Fatalf("New(missing) error = %v, want the real error, naming the path", err)
|
t.Fatalf("New(missing) error = %v, want the real error, naming the path", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewRejectsAnEmptyDataPath(t *testing.T) {
|
func TestNewRejectsAnEmptyDataPath(t *testing.T) {
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(""))
|
_, err := New(WithSeed(1), WithDataPath(""))
|
||||||
if err == nil || !strings.Contains(err.Error(), "empty") {
|
if err == nil || !strings.Contains(err.Error(), "empty") {
|
||||||
t.Fatalf("New(WithDataPath(\"\")) = %v, want the empty path named", err)
|
t.Fatalf("New(WithDataPath(\"\")) = %v, want the empty path named", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewReportsAnEntropyFailure(t *testing.T) {
|
func TestNewWithoutASeedFailsNamingWithSeed(t *testing.T) {
|
||||||
saved := randomBytes
|
for name, opts := range map[string][]Option{"no option": nil, "only data": {WithDataFS()}} {
|
||||||
randomBytes = func([]byte) (int, error) { return 0, errors.New("no entropy") }
|
if _, err := New(opts...); err == nil || !strings.Contains(err.Error(), "WithSeed(") {
|
||||||
defer func() { randomBytes = saved }()
|
t.Errorf("New(%s) = %v, want it refused, naming WithSeed", name, err)
|
||||||
_, err := New(WithoutShippedData(), WithDataFS(fstest.MapFS{"w.json": {Data: []byte(`"x"`)}}))
|
}
|
||||||
if err == nil || !strings.Contains(err.Error(), "no entropy") {
|
}
|
||||||
t.Fatalf("New() without entropy = %v, want the failure reported", err)
|
if _, err := New(WithDataFS(nil)); err == nil || !strings.Contains(err.Error(), "WithSeed(") || !strings.Contains(err.Error(), "module 1 is nil") {
|
||||||
|
t.Errorf("New(WithDataFS(nil)) = %v, want both the missing seed and the nil module named", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWithSeedIsDeterministic(t *testing.T) {
|
func TestWithSeedIsDeterministic(t *testing.T) {
|
||||||
a, b := newGenerator(t, "data", WithSeed(42)), newGenerator(t, "data", WithSeed(42))
|
a, b := shipped(t, WithSeed(42)), shipped(t, WithSeed(42))
|
||||||
for i := 0; i < 50; i++ {
|
for i := 0; i < 50; i++ {
|
||||||
if x, y := fake(t, a, "sv_SE.person"), fake(t, b, "sv_SE.person"); x != y {
|
if x, y := fake(t, a, "sv_SE.person"), fake(t, b, "sv_SE.person"); x != y {
|
||||||
t.Fatalf("same seed diverged at %d: %q != %q", i, x, y)
|
t.Fatalf("same seed diverged at %d: %q != %q", i, x, y)
|
||||||
@@ -73,7 +92,7 @@ func TestWithSeedIsDeterministic(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestDifferentSeedsDiffer(t *testing.T) {
|
func TestDifferentSeedsDiffer(t *testing.T) {
|
||||||
a, b := newGenerator(t, "data", WithSeed(1)), newGenerator(t, "data", WithSeed(2))
|
a, b := shipped(t, WithSeed(1)), shipped(t, WithSeed(2))
|
||||||
for i := 0; i < 50; i++ {
|
for i := 0; i < 50; i++ {
|
||||||
if fake(t, a, "en_US.person") != fake(t, b, "en_US.person") {
|
if fake(t, a, "en_US.person") != fake(t, b, "en_US.person") {
|
||||||
return
|
return
|
||||||
@@ -92,3 +111,43 @@ func fakeTemplate(t *testing.T, f *Generator, s string) string {
|
|||||||
}
|
}
|
||||||
return got
|
return got
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// engine builds a seeded generator with no loaded categories, for rendering tests.
|
||||||
|
func engine(seed uint64) *Generator {
|
||||||
|
return &Generator{drawState: drawstate.New(seed)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// parse unmarshals a JSON template fragment into its dynamic form.
|
||||||
|
func parse(t *testing.T, s string) jsonvalue.Value {
|
||||||
|
t.Helper()
|
||||||
|
v, err := jsonvalue.Decode([]byte(s))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse %q: %v", s, err)
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolved compiles a JSON fragment and resolves it against no data.
|
||||||
|
func resolved(t *testing.T, s string) (node, error) {
|
||||||
|
t.Helper()
|
||||||
|
n, err := compile(parse(t, s))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return n, resolveInlineTemplates(inlineNodes(n, "template"), nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// compiled parses, compiles and resolves a JSON fragment into a node.
|
||||||
|
func compiled(t *testing.T, s string) node {
|
||||||
|
t.Helper()
|
||||||
|
n, err := resolved(t, s)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compile %q: %v", s, err)
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustRender(t *testing.T, f *Generator, s string) string {
|
||||||
|
t.Helper()
|
||||||
|
return renderOnce(f.drawState, compiled(t, s))
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,205 @@
|
|||||||
|
package fejkdata
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata/internal/builtinfunc"
|
||||||
|
"github.com/larvit/fejkdata/internal/grammar"
|
||||||
|
)
|
||||||
|
|
||||||
|
// checkFunc validates a call at compile time: a known builtin, its args, and, for calc, the
|
||||||
|
// operands held in fields.
|
||||||
|
func checkFunc(tok grammar.Token, fields map[string]node) error {
|
||||||
|
err := builtinfunc.Check(tok.Fn, tok.Args)
|
||||||
|
if err == nil && tok.Fn == builtinfunc.CalcName {
|
||||||
|
err = checkCalcFields(tok.Args, fields)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("token {%s}: %w", tok.Body, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseChecked(format string, fields map[string]node) ([]grammar.Token, []unboundRead, error) {
|
||||||
|
toks, err := grammar.ParseFormat(format)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
unbound, err := checkTokens(toks, fields)
|
||||||
|
return toks, unbound, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// unboundRead is a token reading a head no field holds, which only a name bound around the
|
||||||
|
// template can answer; err is the refusal where none does, and body is the token's.
|
||||||
|
type unboundRead struct {
|
||||||
|
head, body string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkTokens proves every token's grammar, functions and field paths, and returns the reads
|
||||||
|
// whose head no field holds, for bindNames to look up among the names.
|
||||||
|
func checkTokens(toks []grammar.Token, fields map[string]node) ([]unboundRead, error) {
|
||||||
|
var unbound []unboundRead
|
||||||
|
for _, t := range toks {
|
||||||
|
u, err := checkToken(t, fields)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
unbound = append(unbound, u...)
|
||||||
|
}
|
||||||
|
return unbound, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkToken(t grammar.Token, fields map[string]node) ([]unboundRead, error) {
|
||||||
|
switch t.Kind {
|
||||||
|
case grammar.BuiltinCall:
|
||||||
|
if err := checkFunc(t, fields); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return checkReads(t, fields, true)
|
||||||
|
case grammar.NameBind:
|
||||||
|
return nil, checkBind(t, fields)
|
||||||
|
case grammar.PathRead:
|
||||||
|
return checkReads(t, fields, false)
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkReads proves each name t reads, an arm or an operand, is a reference or a path into a
|
||||||
|
// field, returning those whose head no field holds.
|
||||||
|
func checkReads(t grammar.Token, fields map[string]node, operands bool) ([]unboundRead, error) {
|
||||||
|
var unbound []unboundRead
|
||||||
|
names := tokenReads(t)
|
||||||
|
for _, name := range names {
|
||||||
|
if grammar.IsRef(name) {
|
||||||
|
if _, _, err := grammar.RefShape(name); err != nil {
|
||||||
|
return nil, fmt.Errorf("token {%s}: %w", t.Body, err)
|
||||||
|
}
|
||||||
|
continue // resolveRefs checks its target once the tree is assembled
|
||||||
|
}
|
||||||
|
missing, err := checkArm(name, fields, !operands && len(names) == 1)
|
||||||
|
if err == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
err = fmt.Errorf("token {%s}: %w", t.Body, err)
|
||||||
|
if !missing {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
unbound = append(unbound, unboundRead{head: splitArm(name, nil).head, body: t.Body, err: err})
|
||||||
|
}
|
||||||
|
return unbound, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkBind proves a {x as name} token is spelled once, binds a reference or a path into a
|
||||||
|
// field, and names a valid name that is no option.
|
||||||
|
func checkBind(t grammar.Token, fields map[string]node) error {
|
||||||
|
ref, name := t.BoundRef, t.Bound
|
||||||
|
if trimmed := strings.TrimSpace(ref) + grammar.AsWord + strings.TrimSpace(name); trimmed != t.Body {
|
||||||
|
return fmt.Errorf("token {%s}: write {%s}", t.Body, trimmed)
|
||||||
|
}
|
||||||
|
if err := checkBound(ref, fields); err != nil {
|
||||||
|
return fmt.Errorf("token {%s}: %w", t.Body, err)
|
||||||
|
}
|
||||||
|
if name == "" {
|
||||||
|
return fmt.Errorf("token {%s}: a binding names nothing; write {%s as n}", t.Body, ref)
|
||||||
|
}
|
||||||
|
if isOption(name) {
|
||||||
|
return fmt.Errorf("token {%s}: %q is an option and can never be a name; rename it", t.Body, name)
|
||||||
|
}
|
||||||
|
if err := grammar.CheckIdentifier(name); err != nil {
|
||||||
|
return fmt.Errorf("token {%s}: name %w", t.Body, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkBound proves what a binding binds is a reference or a path into a field of its binder.
|
||||||
|
func checkBound(ref string, fields map[string]node) error {
|
||||||
|
if grammar.IsRef(ref) {
|
||||||
|
_, _, err := grammar.RefShape(ref)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
missing, err := checkArm(ref, fields, false)
|
||||||
|
if missing {
|
||||||
|
return fmt.Errorf("%w; a binding names a field, or a reference, which starts with /, . or ..", err)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkArm validates one sibling name or path against a template's fields.
|
||||||
|
// wholeToken says the name is the token's entire body, so {/name} would render
|
||||||
|
// the same value and can be offered as the reference spelling. missing says no
|
||||||
|
// field holds the head, which a name may yet answer.
|
||||||
|
func checkArm(name string, fields map[string]node, wholeToken bool) (missing bool, err error) {
|
||||||
|
a := splitArm(name, nil)
|
||||||
|
if err := checkSegments(a); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
field, ok := fields[a.head]
|
||||||
|
if !ok {
|
||||||
|
if a.head == "" {
|
||||||
|
return false, fmt.Errorf("a name is never empty, so this token can name no field")
|
||||||
|
}
|
||||||
|
if isOption(a.head) {
|
||||||
|
return false, fmt.Errorf("%q is an option and can never be a field", a.head)
|
||||||
|
}
|
||||||
|
if len(fields) == 0 {
|
||||||
|
hint := ""
|
||||||
|
if wholeToken && hintableRef(name) {
|
||||||
|
hint = fmt.Sprintf(" — write {/%s} to reference the data", name)
|
||||||
|
}
|
||||||
|
return true, fmt.Errorf("no field %q; a token names a sibling field, and this template has none%s", a.head, hint)
|
||||||
|
}
|
||||||
|
return true, fmt.Errorf("no field %q", a.head)
|
||||||
|
}
|
||||||
|
if err := provePath(field, a.tail, a.head, 0); err != nil {
|
||||||
|
return false, fmt.Errorf("field %q: %w", a.head, err)
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// hintableRef reports whether {/name} is a reference the grammar accepts, so the
|
||||||
|
// hint never names a spelling that fails too.
|
||||||
|
func hintableRef(name string) bool {
|
||||||
|
return grammar.CheckPathIdentifiers(name) == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// tokenReads lists the names t reads: a read's arms, or the operands a call's builtin reads.
|
||||||
|
func tokenReads(t grammar.Token) []string {
|
||||||
|
if t.Kind == grammar.BuiltinCall {
|
||||||
|
return builtinfunc.Operands(t.Fn, t.Args)
|
||||||
|
}
|
||||||
|
return t.Arms
|
||||||
|
}
|
||||||
|
|
||||||
|
// formatOps is a compiled format: its ops, and the size of its literal text.
|
||||||
|
type formatOps struct {
|
||||||
|
ops []op
|
||||||
|
grow int
|
||||||
|
}
|
||||||
|
|
||||||
|
// compileFormat compiles t's parsed format against its resolved references and its names'
|
||||||
|
// targets; a {ref as n} token compiles to no op. checkTokens proved every token valid.
|
||||||
|
func compileFormat(t *template, targets map[*nameBinding]nameTarget) (formatOps, error) {
|
||||||
|
var c formatOps
|
||||||
|
for _, tok := range t.tokens {
|
||||||
|
o := op{Token: tok}
|
||||||
|
var err error
|
||||||
|
switch tok.Kind {
|
||||||
|
case grammar.LiteralRun:
|
||||||
|
c.grow += len(tok.Lit)
|
||||||
|
case grammar.BuiltinCall:
|
||||||
|
o.call = builtinfunc.Prep(tok.Fn, tok.Args)
|
||||||
|
o.operands, err = t.compileArms(builtinfunc.Operands(tok.Fn, tok.Args), targets)
|
||||||
|
case grammar.PathRead:
|
||||||
|
o.arms, err = t.compileArms(tok.Arms, targets)
|
||||||
|
case grammar.NameBind:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return c, fmt.Errorf("token {%s}: %w", tok.Body, err)
|
||||||
|
}
|
||||||
|
c.ops = append(c.ops, o)
|
||||||
|
}
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
package fejkdata
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
// TestSeededOutputIsStable pins the exact bytes a seeded generator emits for each
|
||||||
|
// piece of the token grammar, so a change to how a format is scanned or rendered
|
||||||
|
// cannot quietly shift the rng stream that reproducibility depends on.
|
||||||
|
func TestSeededOutputIsStable(t *testing.T) {
|
||||||
|
dir := writeFiles(t, map[string]string{
|
||||||
|
"alt.json": `{"format":"{a|b}","a":"A","b":"B"}`,
|
||||||
|
"calc.json": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00","100.00"],"qty":["2","3","7"]}`,
|
||||||
|
"classes.json": `"{digits(2)}-{int(1,9)}{int(1,9)}-{upper(2)}-{lower(2)}"`,
|
||||||
|
"escapes.json": `{"format":"01Aa#{x}","x":"!"}`,
|
||||||
|
"funcs.json": `"{hex(6)} {int(10,99)} {float(0,1,3)} {nanoid(5)} {seq()}"`,
|
||||||
|
"municipality.json": `{"format":"{name}","rows":"municipality.tsv","key":"code","name":"name","parent":"region"}`,
|
||||||
|
"municipality.tsv": "code\tname\tregion\n0180\tStockholm\t01\n0184\tSolna\t01\n1280\tMalmö\t12\n1281\tLund\t12\n",
|
||||||
|
"named.json": `{"format":"{p as q}{q.a}{q.b}","p":[{"format":"x","a":"1","b":"2"},{"format":"y","a":"A","b":"B"},{"format":"z","a":"a","b":"b"}]}`,
|
||||||
|
"nested.json": `{"format":"{outer}","outer":{"format":"{inner}-{digits(2)}","inner":"i"}}`,
|
||||||
|
"paths.json": `{"format":"{p.a}{p.b}","p":[{"format":"x","a":"1","b":"2"},{"format":"y","a":"A","b":"B"},{"format":"z","a":"a","b":"b"}]}`,
|
||||||
|
"ref.json": `"see {/alt}"`,
|
||||||
|
"refpath.json": `"{/region.municipality.name} in {/region.name}"`,
|
||||||
|
"region.json": `{"format":"{name}","rows":"region.tsv","key":"code","name":"name","weight":"people"}`,
|
||||||
|
"region.tsv": "code\tname\tpeople\n01\tStockholms län\t3\n12\tSkåne län\t1\n",
|
||||||
|
"repeat.json": `{"format":"{w}","repeat":4,"separator":",","w":["x","y","z"]}`,
|
||||||
|
"sums.json": `{"format":"9{d}{luhn()} {e}{ean()} {m}{mod11()}","d":"012345678901234","e":"123456789012","m":"12345678"}`,
|
||||||
|
"weights.json": `[{"format":"big","weight":9},"tiny"]`,
|
||||||
|
})
|
||||||
|
want := map[string][]string{
|
||||||
|
"alt": {"A", "A", "B", "A"},
|
||||||
|
"calc": {"100.00 x 3 = 700.00", "5.00 x 7 = 35.00", "5.00 x 3 = 35.00", "100.00 x 2 = 139.93"},
|
||||||
|
"classes": {"49-74-VY-gj", "38-68-RP-xo", "47-68-IB-hs", "91-89-LP-gk"},
|
||||||
|
"escapes": {"01Aa#!", "01Aa#!", "01Aa#!", "01Aa#!"},
|
||||||
|
"funcs": {"0016a2 33 0.649 k4Kwj 1", "a00c07 84 0.175 6UjGv 2", "f70eb8 12 0.390 p5p6g 3", "9048a3 90 0.531 I3Aqv 4"},
|
||||||
|
"named": {"ab", "ab", "AB", "AB"},
|
||||||
|
"nested": {"i-73", "i-23", "i-67", "i-85"},
|
||||||
|
"paths": {"aB", "aB", "ab", "AB"},
|
||||||
|
"paths.p.a": {"A", "a", "a", "A"},
|
||||||
|
"ref": {"see A", "see A", "see A", "see B"},
|
||||||
|
"refpath": {"Solna in Skåne län", "Solna in Stockholms län", "Solna in Stockholms län", "Stockholm in Stockholms län"},
|
||||||
|
"region.municipality[0184].name": {"Solna", "Solna", "Solna", "Solna"},
|
||||||
|
"region[12].municipality.name": {"Malmö", "Malmö", "Lund", "Malmö"},
|
||||||
|
"repeat": {"z,y,z,y", "z,z,y,y", "z,z,x,z", "x,z,y,y"},
|
||||||
|
"sums": {"90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780", "90123456789012348 1234567890124 123456780"},
|
||||||
|
"weights": {"big", "big", "big", "big"},
|
||||||
|
}
|
||||||
|
for path := range want {
|
||||||
|
f := newGenerator(t, dir, WithSeed(42))
|
||||||
|
for i, expect := range want[path] {
|
||||||
|
if got := fake(t, f, path); got != expect {
|
||||||
|
t.Errorf("%s draw %d = %q, want %q", path, i, got, expect)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+132
@@ -0,0 +1,132 @@
|
|||||||
|
package fejkdata
|
||||||
|
|
||||||
|
import (
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestStringIsAFormat(t *testing.T) {
|
||||||
|
f := engine(1)
|
||||||
|
for src, want := range map[string]string{
|
||||||
|
`"Malmö"`: "Malmö",
|
||||||
|
`"100 Main St, Apt 1A #0"`: "100 Main St, Apt 1A #0",
|
||||||
|
`"{{x}}"`: "{x}",
|
||||||
|
} {
|
||||||
|
if got := mustRender(t, f, src); got != want {
|
||||||
|
t.Errorf("render(%s) = %q, want %q", src, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := mustRender(t, f, `"{digits(3)}"`); !regexp.MustCompile(`^[0-9]{3}$`).MatchString(got) {
|
||||||
|
t.Errorf(`render("{digits(3)}") = %q, want three digits`, got)
|
||||||
|
}
|
||||||
|
if _, err := resolved(t, `"{x}"`); err == nil || !strings.Contains(err.Error(), `no field "x"`) {
|
||||||
|
t.Errorf(`compile("{x}") = %v, want a no-field error`, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTextIsLiteral(t *testing.T) {
|
||||||
|
if got := mustRender(t, engine(1), `{"format":"100 Main St #1 {x}","x":"A"}`); got != "100 Main St #1 A" {
|
||||||
|
t.Fatalf("text = %q, want it verbatim", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBraceEscapes(t *testing.T) {
|
||||||
|
f := engine(1)
|
||||||
|
for src, want := range map[string]string{
|
||||||
|
`{"format":"{{","x":"v"}`: "{",
|
||||||
|
`{"format":"}}","x":"v"}`: "}",
|
||||||
|
`{"format":"{{x}}","x":"v"}`: "{x}",
|
||||||
|
`{"format":"{{{x}}}","x":"v"}`: "{v}",
|
||||||
|
`{"format":"a{{{{b}}}}","x":"v"}`: "a{{b}}",
|
||||||
|
} {
|
||||||
|
if got := mustRender(t, f, src); got != want {
|
||||||
|
t.Errorf("render(%s) = %q, want %q", src, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for src, want := range map[string]string{
|
||||||
|
`"x}y"`: "}}",
|
||||||
|
`"}"`: "}}",
|
||||||
|
`{"format":"{a{b}","a":"Q"}`: "'{'",
|
||||||
|
`"{x"`: "unterminated",
|
||||||
|
} {
|
||||||
|
if _, err := resolved(t, src); err == nil || !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("compile(%s) = %v, want an error mentioning %s", src, err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTokenSubstitution(t *testing.T) {
|
||||||
|
if got := mustRender(t, engine(1), `{"format":"{x}sson","x":"Erik"}`); got != "Eriksson" {
|
||||||
|
t.Fatalf("token = %q, want Eriksson", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAlternationDrawsOneField(t *testing.T) {
|
||||||
|
f := engine(3)
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for i := 0; i < 100; i++ {
|
||||||
|
seen[mustRender(t, f, `{"format":"{a|b}","a":"A","b":"B"}`)] = true
|
||||||
|
}
|
||||||
|
if !seen["A"] || !seen["B"] || len(seen) != 2 {
|
||||||
|
t.Fatalf("alternation produced %v, want both A and B", seen)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAlternationThreeWay(t *testing.T) {
|
||||||
|
f := engine(4)
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for i := 0; i < 200; i++ {
|
||||||
|
seen[mustRender(t, f, `{"format":"{a|b|c}","a":"A","b":"B","c":"C"}`)] = true
|
||||||
|
}
|
||||||
|
if !seen["A"] || !seen["B"] || !seen["C"] || len(seen) != 3 {
|
||||||
|
t.Fatalf("3-way alternation produced %v, want A, B and C", seen)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHashIsLiteral(t *testing.T) {
|
||||||
|
f := engine(1)
|
||||||
|
cases := map[string]string{
|
||||||
|
`{"format":"","x":"v"}`: "",
|
||||||
|
`{"format":"#","x":"v"}`: "#",
|
||||||
|
`{"format":"##","x":"v"}`: "##",
|
||||||
|
`{"format":"#0#1#A#a","x":"v"}`: "#0#1#A#a",
|
||||||
|
`{"format":"#{x}","x":"v"}`: "#v",
|
||||||
|
}
|
||||||
|
for tmpl, want := range cases {
|
||||||
|
if got := mustRender(t, f, tmpl); got != want {
|
||||||
|
t.Errorf("render(%s) = %q, want %q", tmpl, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultibyteFormat(t *testing.T) {
|
||||||
|
// Scanning is rune-aware: multibyte literals coexist with class chars and
|
||||||
|
// tokens without corrupting indices.
|
||||||
|
got := mustRender(t, engine(2), `{"format":"Öster{x}-{digits(1)}å","x":"väg"}`)
|
||||||
|
if !regexp.MustCompile(`^Österväg-[0-9]å$`).MatchString(got) {
|
||||||
|
t.Fatalf("multibyte format = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFormatCompileErrors(t *testing.T) {
|
||||||
|
for _, bad := range []string{
|
||||||
|
`"{x"`, // unterminated brace
|
||||||
|
`"x}y"`, // a lone } must be written }}
|
||||||
|
`{"format":"{a{b}","a":"Q"}`, // a brace inside a token
|
||||||
|
`"{x}"`, // a bare string has no fields to name
|
||||||
|
`{"format":"{y}","x":"Q"}`, // token names a missing field
|
||||||
|
`"{}"`, // empty token name
|
||||||
|
`{"format":"{a|}","a":"Q"}`, // empty alternation segment
|
||||||
|
`"{luhn(}"`, // malformed function token
|
||||||
|
`"{lower()}"`, // wrong arity
|
||||||
|
`"{nope()}"`, // unknown function
|
||||||
|
`"{luhn(x)}"`, // function given args it takes none of
|
||||||
|
`"{int(1)}"`, // wrong arity
|
||||||
|
`"{float(1,2)}"`, // wrong arity
|
||||||
|
} {
|
||||||
|
if _, err := resolved(t, bad); err == nil {
|
||||||
|
t.Errorf("compile(%s) = nil error, want error", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,3 +1,3 @@
|
|||||||
module gitea.larvit.se/larvit/fejkdata
|
module github.com/larvit/fejkdata
|
||||||
|
|
||||||
go 1.22
|
go 1.22
|
||||||
|
|||||||
@@ -2,128 +2,99 @@ package fejkdata
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"slices"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata/internal/invariant"
|
||||||
)
|
)
|
||||||
|
|
||||||
// walkNodes calls fn once per contained node, passing the dot path that reaches it,
|
// eachNode visits n and every node inside it once, and passes fn each node with the label an
|
||||||
// visiting keys in sorted order so which of several broken nodes gets reported does
|
// error uses for it: the dot path reaching it, then ", line N" for a table cell. It never
|
||||||
// not depend on map iteration.
|
// crosses a reference edge — a {/path} reference is skipped — so a single inline node is
|
||||||
func walkNodes(root map[string]node, fn func(path string, n node) error) error {
|
// walked on its own.
|
||||||
for _, name := range sortedNames(root) {
|
func eachNode(n node, label string, fn func(label string, n node) error) error {
|
||||||
if err := eachNode(root[name], name, fn); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// eachNode visits n and every node contained within it once, passing the dot path
|
|
||||||
// that reaches each. It never crosses a reference edge — a bound {/path} field is
|
|
||||||
// skipped — so a single inline node is walked on its own.
|
|
||||||
func eachNode(n node, path string, fn func(path string, n node) error) error {
|
|
||||||
seen := map[node]bool{}
|
seen := map[node]bool{}
|
||||||
var visit func(string, node) error
|
var visit func(string, node) error
|
||||||
visit = func(path string, m node) error {
|
visit = func(label string, m node) error {
|
||||||
if m == nil || seen[m] {
|
if m == nil || seen[m] {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
seen[m] = true
|
seen[m] = true
|
||||||
if err := fn(path, m); err != nil {
|
if err := fn(label, m); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
for _, c := range contained(m) {
|
for _, c := range contained(m) {
|
||||||
if err := visit(join(path, c.name), c.node); err != nil {
|
if err := visit(c.labelIn(label), c.node); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return visit(path, n)
|
return visit(label, n)
|
||||||
}
|
}
|
||||||
|
|
||||||
// namedNode is a contained child and the segment reaching it; a choice's items carry
|
// containedNode is a contained child and the segment reaching it; a choice's items carry
|
||||||
// no segment, matching how a dot path steps over a choice.
|
// no segment, matching how a dot path steps over a choice.
|
||||||
type namedNode struct {
|
type containedNode struct {
|
||||||
name string
|
name string
|
||||||
node node
|
node node
|
||||||
|
line int // a table cell's line in its rows file, else 0
|
||||||
}
|
}
|
||||||
|
|
||||||
func contained(n node) []namedNode {
|
func (c containedNode) labelIn(label string) string {
|
||||||
|
if c.line > 0 {
|
||||||
|
return fmt.Sprintf("%s, line %d", label, c.line)
|
||||||
|
}
|
||||||
|
return join(label, c.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func contained(n node) []containedNode {
|
||||||
switch n := n.(type) {
|
switch n := n.(type) {
|
||||||
case *folder:
|
case *folder:
|
||||||
return named(n.children)
|
return containedByName(n.children)
|
||||||
case *choice:
|
case *choice:
|
||||||
out := make([]namedNode, len(n.items))
|
out := make([]containedNode, len(n.items))
|
||||||
for i, it := range n.items {
|
for i, it := range n.items {
|
||||||
out[i] = namedNode{node: it}
|
out[i] = containedNode{node: it}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
case *template:
|
case *template:
|
||||||
return named(n.fields)
|
return containedByName(n.fields)
|
||||||
case *table:
|
case *table:
|
||||||
return append([]namedNode{{node: n.format}}, named(n.fields)...)
|
return append([]containedNode{{node: n.formatTemplate}}, containedByName(n.formatTemplate.fields)...)
|
||||||
case *column:
|
case *tableColumn:
|
||||||
if len(n.t.tokens) == 0 {
|
if len(n.t.cellTemplates) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
var out []namedNode
|
var out []containedNode
|
||||||
for r := 0; r < n.t.rows(); r++ {
|
for r := 0; r < n.t.rows.Len(); r++ {
|
||||||
if cell := n.t.cellNode(r, n.i); cell != nil {
|
if cell := n.t.cellTemplate(r, n.i); cell != nil {
|
||||||
out = append(out, namedNode{node: cell})
|
out = append(out, containedNode{node: cell, line: r + 2})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
default:
|
case *nullItem, *tableRow:
|
||||||
return nil
|
return nil
|
||||||
|
default:
|
||||||
|
panic(invariant.Broken("contained has no case for node %T", n))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// named skips a bound {/path} key: it is a render edge, not containment, so using
|
func containedByName(m map[string]node) []containedNode {
|
||||||
// it as a path segment would report a node under a path that does not reach it. Only
|
out := make([]containedNode, 0, len(m))
|
||||||
// a template's fields hold bindings — loadDir skips a dot-prefixed entry, so a
|
|
||||||
// folder's children never carry the prefix — so this one skip serves both.
|
|
||||||
func named(m map[string]node) []namedNode {
|
|
||||||
out := make([]namedNode, 0, len(m))
|
|
||||||
for _, name := range sortedNames(m) {
|
for _, name := range sortedNames(m) {
|
||||||
if isRef(name) {
|
out = append(out, containedNode{name: name, node: m[name]})
|
||||||
continue
|
|
||||||
}
|
|
||||||
out = append(out, namedNode{name: name, node: m[name]})
|
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func sortedNames(m map[string]node) []string {
|
|
||||||
names := make([]string, 0, len(m))
|
|
||||||
for name := range m {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
return names
|
|
||||||
}
|
|
||||||
|
|
||||||
// renderEdge is a child a node renders into, labelled by what reaches it (a field
|
// renderEdge is a child a node renders into, labelled by what reaches it (a field
|
||||||
// name, reference, or choice index) for a readable cycle report. operand names
|
// name, reference, or choice index) for a readable cycle report.
|
||||||
// the builtin when the label is its operand rather than a token, so an error can
|
|
||||||
// name it the way the author wrote it.
|
|
||||||
type renderEdge struct {
|
type renderEdge struct {
|
||||||
to node
|
to node
|
||||||
label string
|
label string
|
||||||
operand string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// reached names an edge as the author spelled it, the vocabulary boundReaders uses
|
// renderEdges lists the children rendering n recurses into, mirroring render.
|
||||||
// for the sibling fence.
|
|
||||||
func (e renderEdge) reached() string {
|
|
||||||
if e.operand != "" {
|
|
||||||
return fmt.Sprintf("%s operand %q", e.operand, e.label)
|
|
||||||
}
|
|
||||||
return "{" + e.label + "}"
|
|
||||||
}
|
|
||||||
|
|
||||||
// renderEdges lists the children rendering n recurses into, mirroring expand: a
|
|
||||||
// choice's items, and a template's field/reference tokens plus its operands. A
|
|
||||||
// folder renders nothing, so it has no edges.
|
|
||||||
func renderEdges(n node) []renderEdge {
|
func renderEdges(n node) []renderEdge {
|
||||||
switch n := n.(type) {
|
switch n := n.(type) {
|
||||||
case *choice:
|
case *choice:
|
||||||
@@ -134,120 +105,40 @@ func renderEdges(n node) []renderEdge {
|
|||||||
return es
|
return es
|
||||||
case *template:
|
case *template:
|
||||||
var es []renderEdge
|
var es []renderEdge
|
||||||
add := func(name, operand string) {
|
for _, o := range n.compiled.ops {
|
||||||
a := splitArm(name, n.refs)
|
for _, a := range slices.Concat(o.operands, o.arms) {
|
||||||
c, ok := n.fields[a.key]
|
for _, leaf := range a.leaves {
|
||||||
if !ok {
|
es = append(es, renderEdge{leaf, a.spelling})
|
||||||
return
|
}
|
||||||
}
|
|
||||||
for _, leaf := range pathLeaves(c, a.tail) {
|
|
||||||
es = append(es, renderEdge{leaf, name, operand})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ = eachToken(n.format, func(t ftoken) error {
|
|
||||||
if t.kind != 'b' {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if fn, _, isFunc := funcCall(t.body); isFunc {
|
|
||||||
for _, operand := range tokenOperands(t.body) {
|
|
||||||
add(operand, fn)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
for _, name := range splitOutside(t.body, '|') {
|
|
||||||
add(name, "")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
return es
|
return es
|
||||||
case *table:
|
case *table:
|
||||||
return []renderEdge{{to: n.format, label: "format"}}
|
return []renderEdge{{to: n.formatTemplate, label: "format"}}
|
||||||
case *row:
|
case *tableRow:
|
||||||
return []renderEdge{{to: n.t.format, label: "format"}}
|
return []renderEdge{{to: n.t.formatTemplate, label: "format"}}
|
||||||
case *column:
|
case *tableColumn:
|
||||||
var es []renderEdge
|
var es []renderEdge
|
||||||
for _, c := range contained(n) {
|
for _, c := range contained(n) {
|
||||||
es = append(es, renderEdge{to: c.node, label: n.t.columns[n.i]})
|
es = append(es, renderEdge{to: c.node, label: n.name()})
|
||||||
}
|
}
|
||||||
return es
|
return es
|
||||||
default:
|
case *folder, *nullItem:
|
||||||
return nil
|
return nil
|
||||||
|
default:
|
||||||
|
panic(invariant.Broken("renderEdges has no case for node %T", n))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// pathLeaves lists what a token's dotted tail renders: a choice on the way
|
type renderCounts map[node]int
|
||||||
// contributes every variant, since any of them may be the one drawn. checkPath has
|
|
||||||
// already proved the tail resolves in every variant.
|
|
||||||
func pathLeaves(n node, tail []string) []node {
|
|
||||||
var out []node
|
|
||||||
_, _ = walkPath(n, tail, pathWalk{
|
|
||||||
choice: func(c *choice, _ []string) ([]node, error) { return c.items, nil },
|
|
||||||
leaf: func(n node) error { out = append(out, n); return nil },
|
|
||||||
})
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// nodeScope is the set of nodes one validation pass covers: a whole loaded tree,
|
func (m renderCounts) renderCount(n node) int {
|
||||||
// or a single inline node.
|
|
||||||
type nodeScope func(fn func(path string, n node) error) error
|
|
||||||
|
|
||||||
func treeScope(root map[string]node) nodeScope {
|
|
||||||
return func(fn func(path string, n node) error) error { return walkNodes(root, fn) }
|
|
||||||
}
|
|
||||||
|
|
||||||
func inlineScope(n node, label string) nodeScope {
|
|
||||||
return func(fn func(path string, m node) error) error { return eachNode(n, label, fn) }
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkScope(s nodeScope) error {
|
|
||||||
if err := checkColumns(s); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return checkRenders(s)
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkRenders runs the per-node fences over a scope, each over the whole scope
|
|
||||||
// before the next, so which of several broken nodes is reported does not depend on
|
|
||||||
// the walk. It runs after checkNoCycles, whose guarantee is what lets the walks
|
|
||||||
// terminate.
|
|
||||||
func checkRenders(s nodeScope) error {
|
|
||||||
mem := reachMemo{}
|
|
||||||
if err := s(func(path string, n node) error { return repeatCheck(path, n, mem) }); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := s(heldCheck); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fence := &drawCheck{}
|
|
||||||
refs := false
|
|
||||||
if err := s(func(path string, n node) error {
|
|
||||||
if t, ok := n.(*template); ok && len(t.refs) > 0 {
|
|
||||||
refs = true
|
|
||||||
}
|
|
||||||
return fence.checkDrawGroup(path, n)
|
|
||||||
}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if refs {
|
|
||||||
if err := s(fence.checkDraws); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := s(fence.checkRecordDraws); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return s((&valueProof{}).checkDatatype)
|
|
||||||
}
|
|
||||||
|
|
||||||
type reachMemo map[node]int
|
|
||||||
|
|
||||||
func (m reachMemo) of(n node) int {
|
|
||||||
if r, done := m[n]; done {
|
if r, done := m[n]; done {
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
r := 1
|
r := 1
|
||||||
for _, e := range renderEdges(n) {
|
for _, e := range renderEdges(n) {
|
||||||
if c := m.of(e.to); c > r {
|
if c := m.renderCount(e.to); c > r {
|
||||||
r = c
|
r = c
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -260,9 +151,10 @@ func (m reachMemo) of(n node) int {
|
|||||||
|
|
||||||
// repeatCheck bounds the renders a repeat multiplies to along any root-to-leaf
|
// repeatCheck bounds the renders a repeat multiplies to along any root-to-leaf
|
||||||
// path, so nested repeats cannot build what one repeat may not.
|
// path, so nested repeats cannot build what one repeat may not.
|
||||||
func repeatCheck(path string, n node, mem reachMemo) error {
|
// docs/decisions.md#the-repeat-cap-bounds-renders-not-bytes
|
||||||
if t, ok := n.(*template); ok && t.repeat > 1 && mem.of(n) > MaxRepeat {
|
func repeatCheck(label string, n node, mem renderCounts) error {
|
||||||
return fmt.Errorf("%s: repeat %d multiplies to %d renders along one path, above the maximum %d", path, t.repeat, mem.of(n), MaxRepeat)
|
if t, ok := n.(*template); ok && t.repeat > 1 && mem.renderCount(n) > MaxRepeat {
|
||||||
|
return fmt.Errorf("%s: repeat %d multiplies to %d renders along one path, above the maximum %d", label, t.repeat, mem.renderCount(n), MaxRepeat)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -270,32 +162,42 @@ func repeatCheck(path string, n node, mem reachMemo) error {
|
|||||||
// checkNoCycles rejects a reference cycle: a node whose rendering can reach itself
|
// checkNoCycles rejects a reference cycle: a node whose rendering can reach itself
|
||||||
// — directly, mutually, or through a chain — never terminates, so it must fail at
|
// — directly, mutually, or through a chain — never terminates, so it must fail at
|
||||||
// New rather than stack-overflow at render. It is a depth-first walk of the render
|
// New rather than stack-overflow at render. It is a depth-first walk of the render
|
||||||
// graph (renderEdges); grey marks nodes on the current path so a back-edge to one
|
// graph (renderEdges). Every node is a root: a field its parent's format never
|
||||||
// is the cycle, while black lets a shared node (a DAG, not a cycle) be skipped.
|
// renders is still reachable by dot path, so a cycle in one would otherwise reach
|
||||||
// Every node is a root: a field its parent's format never renders is still reachable
|
// render and be fatal there.
|
||||||
// by dot path, so a cycle in one would otherwise reach render and be fatal there.
|
func checkNoCycles(s nodeSet) error {
|
||||||
func checkNoCycles(root map[string]node) error {
|
|
||||||
const (
|
const (
|
||||||
grey = 1
|
grey = 1
|
||||||
black = 2
|
black = 2
|
||||||
)
|
)
|
||||||
|
inSet := map[node]bool{}
|
||||||
|
_ = s(func(_ string, n node) error {
|
||||||
|
inSet[n] = true
|
||||||
|
if t, isTable := n.(*table); isTable {
|
||||||
|
inSet[t.rowNode] = true // a selector lands on it, and contained lists it nowhere
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
color := map[node]int{}
|
color := map[node]int{}
|
||||||
var visit func(n node, path string) error
|
var visit func(n node, label string) error
|
||||||
visit = func(n node, path string) error {
|
visit = func(n node, label string) error {
|
||||||
|
if !inSet[n] {
|
||||||
|
return nil // loaded before these nodes and proven then, so it never reaches back into it
|
||||||
|
}
|
||||||
switch color[n] {
|
switch color[n] {
|
||||||
case grey:
|
case grey:
|
||||||
return fmt.Errorf("reference cycle: %s", path)
|
return fmt.Errorf("reference cycle: %s", label)
|
||||||
case black:
|
case black:
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
color[n] = grey
|
color[n] = grey
|
||||||
for _, e := range renderEdges(n) {
|
for _, e := range renderEdges(n) {
|
||||||
if err := visit(e.to, path+" -> "+e.label); err != nil {
|
if err := visit(e.to, label+" -> "+e.label); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
color[n] = black
|
color[n] = black
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return walkNodes(root, func(path string, n node) error { return visit(n, path) })
|
return s(func(label string, n node) error { return visit(n, label) })
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,358 +0,0 @@
|
|||||||
package fejkdata
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// heldCheck rejects every route to a held sibling name except the ones that read its
|
|
||||||
// draw. An expansion holds one draw of that name; anything else that renders it draws
|
|
||||||
// again, and the two disagree. checkNoOverlap settles the spellings within one format
|
|
||||||
// (a token, an operand); this settles the rest — a reference, whether it sits in that
|
|
||||||
// format or in anything the format renders, however deep. A reference path is held
|
|
||||||
// for the whole render instead, which drawCheck fences.
|
|
||||||
func heldCheck(path string, n node) error {
|
|
||||||
t, ok := n.(*template)
|
|
||||||
if !ok || len(t.held) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
readers := boundReaders(t.format, t.bound, t.refs)
|
|
||||||
for _, head := range heldHeads(t) {
|
|
||||||
if _, isPath := t.bound[head]; isPath && isRef(head) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := checkHeadHeld(t, head, readers); err != nil {
|
|
||||||
return fmt.Errorf("%s: %w", path, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// heldHeads lists a template's held names, operand heads first, then paths, each
|
|
||||||
// in name order: a level read both ways is reported by the operand's fence, and
|
|
||||||
// which overlap is reported does not vary.
|
|
||||||
func heldHeads(t *template) []string {
|
|
||||||
heads := make([]string, 0, len(t.held))
|
|
||||||
for head := range t.held {
|
|
||||||
heads = append(heads, head)
|
|
||||||
}
|
|
||||||
sort.Slice(heads, func(i, j int) bool {
|
|
||||||
_, pi := t.bound[heads[i]]
|
|
||||||
_, pj := t.bound[heads[j]]
|
|
||||||
if pi != pj {
|
|
||||||
return !pi
|
|
||||||
}
|
|
||||||
return heads[i] < heads[j]
|
|
||||||
})
|
|
||||||
return heads
|
|
||||||
}
|
|
||||||
|
|
||||||
// pinned collects what the hold of head answers for: a path pins the levels it
|
|
||||||
// passes through and the leaf it lands on, an operand exactly the value its render
|
|
||||||
// produces.
|
|
||||||
func pinned(t *template, head string, readers []reader) map[node]bool {
|
|
||||||
held := map[node]bool{}
|
|
||||||
if _, isPath := t.bound[head]; !isPath {
|
|
||||||
operandDraw(t.fields[head], held)
|
|
||||||
return held
|
|
||||||
}
|
|
||||||
for _, r := range readers {
|
|
||||||
if a := splitArm(r.name, t.refs); a.key == head {
|
|
||||||
coverPath(t.fields[head], a.tail, held)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return held
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkHeadHeld rejects every route to what head's hold pins except the readers
|
|
||||||
// holding it. One seen set across the edges: a node that cannot reach the level
|
|
||||||
// cannot reach it by another route either, so it is walked once.
|
|
||||||
func checkHeadHeld(t *template, head string, readers []reader) error {
|
|
||||||
held := pinned(t, head, readers)
|
|
||||||
if len(held) == 0 {
|
|
||||||
return nil // a fixed head holds nothing to reach
|
|
||||||
}
|
|
||||||
reader, isPath := t.bound[head]
|
|
||||||
seen := map[node]bool{}
|
|
||||||
for _, e := range renderEdges(t) {
|
|
||||||
if splitArm(e.label, t.refs).key == head || !renders(e.to, held, seen) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if isPath {
|
|
||||||
return fmt.Errorf("%s renders %q, which {%s} reads a path into; name the fields you want instead", e.reached(), head, reader)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("%s renders %q, which a {%s()} also reads; reach it one way so it is drawn once", e.reached(), head, operandReader(t, head))
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// operandReader names the builtin whose operand holds head.
|
|
||||||
func operandReader(t *template, head string) string {
|
|
||||||
fn := ""
|
|
||||||
_ = eachToken(t.format, func(tok ftoken) error {
|
|
||||||
if tok.kind != 'b' || fn != "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if name, _, isFunc := funcCall(tok.body); isFunc {
|
|
||||||
for _, operand := range tokenOperands(tok.body) {
|
|
||||||
if splitArm(operand, t.refs).key == head {
|
|
||||||
fn = name
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
return fn
|
|
||||||
}
|
|
||||||
|
|
||||||
// coverPath collects what holding one path pins: every choice level the path
|
|
||||||
// passes through, whole, and the leaf it renders.
|
|
||||||
func coverPath(n node, tail []string, into map[node]bool) {
|
|
||||||
_, _ = walkPath(n, tail, pathWalk{
|
|
||||||
choice: func(c *choice, _ []string) ([]node, error) { cover(c, into, false); return nil, nil },
|
|
||||||
leaf: func(n node) error { cover(n, into, false); return nil },
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// cover collects a level and everything contained in it. A fixed string outside a
|
|
||||||
// choice is left out — it cannot disagree with itself — but inside one each
|
|
||||||
// variant carries its own, so there it counts.
|
|
||||||
func cover(n node, into map[node]bool, inChoice bool) {
|
|
||||||
if isFixed(n) && !inChoice {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
into[n] = true
|
|
||||||
_, isChoice := n.(*choice)
|
|
||||||
for _, c := range contained(n) {
|
|
||||||
cover(c.node, into, inChoice || isChoice)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// operandDraw collects what one held draw of an operand answers for: the operand
|
|
||||||
// and what rendering it settles inside itself. The builtin renders its operand
|
|
||||||
// whole, so that draw fixes every value the render produced, and a second route to
|
|
||||||
// any of them disagrees with it.
|
|
||||||
//
|
|
||||||
// The walk stops at a reference edge, which is where the operand's own value ends
|
|
||||||
// and a shared source begins: two names referencing one category are two draws, the
|
|
||||||
// same rule {word} {word} follows.
|
|
||||||
func operandDraw(n node, into map[node]bool) {
|
|
||||||
if isFixed(n) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if into[n] {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
into[n] = true
|
|
||||||
for _, e := range renderEdges(n) {
|
|
||||||
if isRef(e.label) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
operandDraw(e.to, into)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// isFixed is a string that varies nothing: fixed text with no fields to read into.
|
|
||||||
func isFixed(n node) bool {
|
|
||||||
t, ok := n.(*template)
|
|
||||||
return ok && t.fixed && len(t.fields) == 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// renders reports whether rendering n can reach anything in want, following the
|
|
||||||
// same edges expand does. seen keeps a node shared by several routes from being
|
|
||||||
// walked twice; checkNoCycles has already proved the graph is a DAG, so the walk
|
|
||||||
// ends.
|
|
||||||
func renders(n node, want, seen map[node]bool) bool {
|
|
||||||
if want[n] {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if seen[n] {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
seen[n] = true
|
|
||||||
for _, e := range renderEdges(n) {
|
|
||||||
if renders(e.to, want, seen) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkNoOverlap rejects a format that both renders a sibling level and reads a path
|
|
||||||
// into it — {p} beside {p.first}, {p.addr} beside {p.addr.city}. The path reads the
|
|
||||||
// level's held draw while rendering the level expands it afresh, so their values would
|
|
||||||
// disagree. Reads are compared in sorted order, so which pair is reported does not
|
|
||||||
// depend on where the tokens sit.
|
|
||||||
func checkNoOverlap(format string, bound map[string]string, refs map[string]refBinding) error {
|
|
||||||
names := boundReaders(format, bound, refs)
|
|
||||||
// Stable over one format-order scan, so two readers of one name (a token and a
|
|
||||||
// calc operand both naming "p") are reported as the format writes them.
|
|
||||||
sort.SliceStable(names, func(i, j int) bool { return names[i].path < names[j].path })
|
|
||||||
for i, level := range names {
|
|
||||||
for _, path := range names[i+1:] {
|
|
||||||
if strings.HasPrefix(path.path, level.path+".") {
|
|
||||||
return fmt.Errorf("%s renders a level that {%s} reads a path into; name the fields you want instead", level.label, path.name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// reader is one way a format reaches a bound field: as written, by its one
|
|
||||||
// spelling, and how to name it.
|
|
||||||
type reader struct{ name, path, label string }
|
|
||||||
|
|
||||||
// boundReaders lists every way a format reaches a bound sibling field, in the order the
|
|
||||||
// format writes them. An operand renders its field, so it names a level exactly
|
|
||||||
// as a token does; one scan finds both, which is what puts them in one order.
|
|
||||||
func boundReaders(format string, bound map[string]string, refs map[string]refBinding) []reader {
|
|
||||||
var names []reader
|
|
||||||
_ = eachToken(format, func(t ftoken) error {
|
|
||||||
if t.kind != 'b' {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if fn, _, isFunc := funcCall(t.body); isFunc {
|
|
||||||
for _, operand := range tokenOperands(t.body) {
|
|
||||||
a := splitArm(operand, refs)
|
|
||||||
if _, isBound := bound[a.key]; isBound && !isRef(a.key) {
|
|
||||||
names = append(names, reader{a.name, a.path, fmt.Sprintf("%s operand %q", fn, operand)})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
for _, a := range splitArms(t.body, refs) {
|
|
||||||
if _, isBound := bound[a.key]; isBound && !isRef(a.key) {
|
|
||||||
names = append(names, reader{a.name, a.path, "token {" + a.name + "}"})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
return names
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkNoRepeatedRead rejects a bare token repeated on a held name: {w} {w} beside
|
|
||||||
// {uppercase(w)} would read one draw twice, where {w} {w} alone draws twice. The
|
|
||||||
// error names the single-token spelling.
|
|
||||||
func checkNoRepeatedRead(format string, c formatOps, refs map[string]refBinding) error {
|
|
||||||
count := map[string]int{}
|
|
||||||
return eachToken(format, func(t ftoken) error {
|
|
||||||
if t.kind != 'b' {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if _, _, isFunc := funcCall(t.body); isFunc {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
for _, a := range splitArms(t.body, refs) {
|
|
||||||
if len(a.tail) > 0 || !c.held[a.key] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if count[a.key]++; count[a.key] > 1 {
|
|
||||||
return fmt.Errorf("token {%s} is repeated, and %s holds %q to one draw per expansion; write {%s} once", a.name, c.holder[a.key], a.key, a.name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// draws is what has already been drawn for held names: the variant each was drawn as,
|
|
||||||
// so every path under it reads one row, and the draw each read made, by its one
|
|
||||||
// spelling, so the same read written twice reads one draw. An expansion keeps one for
|
|
||||||
// its sibling names, and a render one per group for its reference paths.
|
|
||||||
type draws struct {
|
|
||||||
variant map[string]node
|
|
||||||
value map[string]draw
|
|
||||||
pins [8]tablePin // the rows pinned, inline so a render over a country's five-deep geo tree stays off the heap
|
|
||||||
npins int
|
|
||||||
more map[*table]int // the rows pinned past the inline eight
|
|
||||||
s *session // what draws a row; nil where a walk only proves selectors
|
|
||||||
}
|
|
||||||
|
|
||||||
// draw is what one read drew: its text, and whether it landed on a null.
|
|
||||||
type draw struct {
|
|
||||||
text string
|
|
||||||
null bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// readField renders one arm of a token. An arm's key is a sibling field or a
|
|
||||||
// reference linkRefs bound into fields. A name the expansion holds — a level some
|
|
||||||
// token addresses by dotted path, or a field an operand reads — is drawn once and
|
|
||||||
// kept, so {place.postal-code} and {place.locality} read one row, either read twice
|
|
||||||
// gives one value, and a shown operand is the operand computed. Every other name is
|
|
||||||
// drawn afresh, so {word} {word} still draws twice. checkTokens, checkPath and
|
|
||||||
// linkRefs prove every step, so the walk cannot fail.
|
|
||||||
func readField(s *session, t *template, held *draws, sc drawScope, a arm) draw {
|
|
||||||
if !t.held[a.key] {
|
|
||||||
if len(a.tail) > 0 {
|
|
||||||
panic(fmt.Sprintf("fejkdata: %q reads a path into %q, which the expansion does not hold", a.name, a.key))
|
|
||||||
}
|
|
||||||
return draw{text: render(s, t.fields[a.key], sc)}
|
|
||||||
}
|
|
||||||
d := readScope(s, held, sc, a)
|
|
||||||
if r, done := d.value[a.path]; done {
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
leaf, err := walkPath(t.fields[a.key], a.tail, pathWalk{
|
|
||||||
// Hold the draw at every level passed through, so two paths sharing a
|
|
||||||
// prefix share it.
|
|
||||||
choice: func(c *choice, rest []string) ([]node, error) {
|
|
||||||
key := a.key
|
|
||||||
if consumed := len(a.tail) - len(rest); consumed > 0 {
|
|
||||||
key = a.steps[consumed-1]
|
|
||||||
}
|
|
||||||
n, drew := d.variant[key]
|
|
||||||
if !drew {
|
|
||||||
n = drawn(s, c)
|
|
||||||
if d.variant == nil {
|
|
||||||
d.variant = map[string]node{}
|
|
||||||
}
|
|
||||||
d.variant[key] = n
|
|
||||||
}
|
|
||||||
return []node{n}, nil
|
|
||||||
},
|
|
||||||
pins: d,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Sprintf("fejkdata: %q: %v; a fence should have refused this at New", a.name, err))
|
|
||||||
}
|
|
||||||
r := renderLeaf(s, leaf, sc)
|
|
||||||
if d.value == nil {
|
|
||||||
d.value = map[string]draw{}
|
|
||||||
}
|
|
||||||
d.value[a.path] = r
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
|
|
||||||
// readScope is the draws a held read keeps its draw in: for a reference that reads a path,
|
|
||||||
// the render's draws for its group, so its draw spans the render; for a sibling, or a
|
|
||||||
// reference read whole, held.
|
|
||||||
func readScope(s *session, held *draws, sc drawScope, a arm) *draws {
|
|
||||||
if isRef(a.key) && len(a.tail) > 0 {
|
|
||||||
return sc.draws(s)
|
|
||||||
}
|
|
||||||
return held
|
|
||||||
}
|
|
||||||
|
|
||||||
// renderLeaf draws and renders what a read lands on: null on a null item, or on a column of one
|
|
||||||
// reference alone whose read drew null.
|
|
||||||
func renderLeaf(s *session, n node, sc drawScope) draw {
|
|
||||||
n = drawn(s, n)
|
|
||||||
if _, isNull := n.(*null); isNull {
|
|
||||||
return draw{null: true}
|
|
||||||
}
|
|
||||||
r := draw{text: render(s, n, sc)}
|
|
||||||
if t, _ := n.(*template); t != nil && t.readsColumn != nil {
|
|
||||||
r.null = sc.in(t).draws(s).value[t.readsColumn.a.path].null
|
|
||||||
}
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
|
|
||||||
// drawn resolves a choice to one variant, so a bound head is a concrete node the
|
|
||||||
// rest of the expansion shares. Nested choices unwrap too: a draw is one value, not
|
|
||||||
// another set to pick from.
|
|
||||||
func drawn(s *session, n node) node {
|
|
||||||
for c, ok := n.(*choice); ok; c, ok = n.(*choice) {
|
|
||||||
n = pick(s, c)
|
|
||||||
}
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
-771
@@ -1,771 +0,0 @@
|
|||||||
package fejkdata
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// This file pins the one-draw rule for dotted sibling tokens. A format that
|
|
||||||
// addresses a sibling by path ({place.locality}) draws that sibling once per
|
|
||||||
// expansion, so every {place.*} token in the same format reads the same variant.
|
|
||||||
// That is what makes a correlated pair — a locality and a postal code that agree
|
|
||||||
// — expressible in data, without the row having to be the whole node.
|
|
||||||
|
|
||||||
// swedishPlaces is a two-variant sibling whose variants pair a locality with the
|
|
||||||
// postal-code prefix that really belongs to it.
|
|
||||||
const swedishPlaces = `{"format":"%s","place":[{"format":"{locality}","locality":"Stockholm","postal-code":"1{digits(2)} {digits(2)}"},{"format":"{locality}","locality":"Tranås","postal-code":"573 {digits(2)}"}]}`
|
|
||||||
|
|
||||||
// agree reports whether a rendered "postcode locality" pair is a real pairing.
|
|
||||||
var agree = regexp.MustCompile(`^(1[0-9]{2} [0-9]{2} Stockholm|573 [0-9]{2} Tranås)$`)
|
|
||||||
|
|
||||||
func places(format string) string {
|
|
||||||
return strings.Replace(swedishPlaces, "%s", format, 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDottedTokenCorrelatesSiblingFields(t *testing.T) {
|
|
||||||
f := engine(1)
|
|
||||||
seen := map[string]bool{}
|
|
||||||
for i := 0; i < 500; i++ {
|
|
||||||
got := mustRender(t, f, places("{place.postal-code} {place.locality}"))
|
|
||||||
if !agree.MatchString(got) {
|
|
||||||
t.Fatalf("draw %d = %q, want a postcode and locality that pair", i, got)
|
|
||||||
}
|
|
||||||
seen[got[len(got)-4:]] = true
|
|
||||||
}
|
|
||||||
if len(seen) < 2 {
|
|
||||||
t.Fatalf("500 draws produced only %v, want both localities", seen)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRenderingALevelAndReadingIntoItIsRejected(t *testing.T) {
|
|
||||||
// One draw, one spelling. A token that renders a level and one that reads a
|
|
||||||
// path into it name the same draw two ways, and only the path reads what the
|
|
||||||
// other rendered — so the pair is a load error rather than a shape whose two
|
|
||||||
// halves can disagree.
|
|
||||||
rejected := map[string]string{
|
|
||||||
"bare head beside a path": `{"format":"{p} + {p.first}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
|
|
||||||
"prefix path beside a path": `{"format":"{p.addr}|{p.addr.city}","p":{"format":"{addr}","addr":[` +
|
|
||||||
`{"format":"{city}","city":["Kiruna","Boden"]},{"format":"{city}","city":["Malmö","Lund"]}]}}`,
|
|
||||||
"either order": `{"format":"{p.first} + {p}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
|
|
||||||
}
|
|
||||||
for name, file := range rejected {
|
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file})))
|
|
||||||
if err == nil {
|
|
||||||
t.Errorf("%s: New = nil error, want the overlapping tokens rejected", name)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "reads a path into") {
|
|
||||||
t.Errorf("%s: New = %v, want it to name the overlap", name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// The same path twice is one spelling, so it stays legal.
|
|
||||||
if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
|
|
||||||
"cat": `{"format":"{p.first} + {p.first}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
|
|
||||||
}))); err != nil {
|
|
||||||
t.Errorf("New = %v, want one path read twice accepted", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCalcOperandNamingABoundLevelIsRejected(t *testing.T) {
|
|
||||||
// A calc operand renders its field, so naming a bound level in one is the same
|
|
||||||
// overlap as a bare token: {calc(item * 1)} renders what {item.price} reads a
|
|
||||||
// path into, and the two disagree.
|
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
|
|
||||||
"cat": `{"format":"{item.price}|{calc(item * 1)}","item":[` +
|
|
||||||
`{"format":"{price}","price":"10"},{"format":"{price}","price":"20"}]}`,
|
|
||||||
})))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "reads a path into") {
|
|
||||||
t.Fatalf("New = %v, want the calc operand rejected as an overlap", err)
|
|
||||||
}
|
|
||||||
// Arithmetic over fields no path names is untouched.
|
|
||||||
if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
|
|
||||||
"cat": `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":"19.99","qty":"3"}`,
|
|
||||||
}))); err != nil {
|
|
||||||
t.Errorf("New = %v, want plain arithmetic accepted", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReferenceNamingABoundLevelIsRejected(t *testing.T) {
|
|
||||||
// A reference can name a bound level from the data root, which renders it
|
|
||||||
// afresh beside the path that reads its held draw — the same overlap by
|
|
||||||
// another spelling.
|
|
||||||
// A category never references itself, so each reference back into cat sits in a
|
|
||||||
// second category cat renders.
|
|
||||||
rejected := map[string]map[string]string{
|
|
||||||
"reference names the head": {
|
|
||||||
"cat": `{"format":"{p.first}|{/hop}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
|
|
||||||
"hop": `"{/cat.p}"`,
|
|
||||||
},
|
|
||||||
"reference names the leaf": {
|
|
||||||
"cat": `{"format":"{p.addr}|{/hop}","p":{"format":"x","addr":["A","B","C","D"]}}`,
|
|
||||||
"hop": `"{/cat.p.addr}"`,
|
|
||||||
},
|
|
||||||
// The reference need not sit in the format that binds: any field it renders
|
|
||||||
// reaches the level just the same, however deep.
|
|
||||||
"reference from a sibling field": {
|
|
||||||
"cat": `{"format":"{p.first}|{inner}","p":[` +
|
|
||||||
`{"format":"{first}-{last}","first":"A","last":"1"},{"format":"{first}-{last}","first":"B","last":"2"}],` +
|
|
||||||
`"inner":"{/hop}"}`,
|
|
||||||
"hop": `"{/cat.p}"`,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for name, files := range rejected {
|
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, files)))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "reads a path into") {
|
|
||||||
t.Errorf("%s: New = %v, want the reference rejected as an overlap", name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// A reference to anything this format does not bind is untouched.
|
|
||||||
if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
|
|
||||||
"cat": `{"format":"{p.first} {/surname}","p":{"format":"{first}","first":["Anna","Bo"]}}`,
|
|
||||||
"surname": `["Eriksson","Lindqvist"]`,
|
|
||||||
}))); err != nil {
|
|
||||||
t.Errorf("New = %v, want a reference outside the bound level accepted", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCycleReachedOnlyByAPathTokenIsRejected(t *testing.T) {
|
|
||||||
// A path token renders what it lands on, not the level it started from, so the
|
|
||||||
// cycle walk has to follow it there. A head whose own format names nothing
|
|
||||||
// would otherwise hide the cycle until render, where it is fatal.
|
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
|
|
||||||
"a": `{"format":"{p.x}","p":{"format":"static","x":"{/b}"}}`,
|
|
||||||
"b": `"{/a}"`,
|
|
||||||
})))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "reference cycle") {
|
|
||||||
t.Fatalf("New = %v, want the cycle through {p.x} rejected", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestALevelRenderedOnlyByAPathTokenIsHeld(t *testing.T) {
|
|
||||||
// {p.a} renders q, so it is a route to the level {q.x} holds — even though p's
|
|
||||||
// own format names nothing.
|
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
|
|
||||||
"thing": `{"format":"{p.a} {q.x}","p":{"format":"static","a":"{/hop}"},` +
|
|
||||||
`"q":{"format":"{x}","x":["1","2"]}}`,
|
|
||||||
"hop": `"{/thing.q}"`,
|
|
||||||
})))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "reads a path into") {
|
|
||||||
t.Fatalf("New = %v, want the second route to q rejected", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestACalcOperandIsHeldAgainstEveryRoute pins what a {calc()} operand's hold
|
|
||||||
// fences. A calc renders its operand whole, so the draw it holds is that one node's
|
|
||||||
// value: another route conflicts only by naming that same node, and then the value
|
|
||||||
// shown is not the value computed. Two names that merely draw from one source are
|
|
||||||
// two draws, as {word} {word} is — the rule everywhere else in the engine.
|
|
||||||
func TestACalcOperandIsHeldAgainstEveryRoute(t *testing.T) {
|
|
||||||
rejected := map[string]struct {
|
|
||||||
files map[string]string
|
|
||||||
want string // the route the error names, spelled as the author wrote it
|
|
||||||
}{
|
|
||||||
// A category never references itself, so each route back into cat sits in a
|
|
||||||
// second category cat renders.
|
|
||||||
"a reference beside the operand": {
|
|
||||||
map[string]string{
|
|
||||||
"cat": `{"format":"{/hop} x 2 = {calc(net * 2, 2)}","net":["10.00","20.00"]}`,
|
|
||||||
"hop": `"{/cat.net}"`,
|
|
||||||
},
|
|
||||||
`{/hop} renders "net"`,
|
|
||||||
},
|
|
||||||
"a reference one level down": {
|
|
||||||
map[string]string{
|
|
||||||
"cat": `{"format":"{calc(net * 2, 2)} {q}","net":["10.00","20.00"],` +
|
|
||||||
`"q":"{/hop}"}`,
|
|
||||||
"hop": `"{/cat.net}"`,
|
|
||||||
},
|
|
||||||
`{q} renders "net"`,
|
|
||||||
},
|
|
||||||
// A reference ending at the operand binds the choice itself, not a variant,
|
|
||||||
// so wrapping the operand in one changes nothing.
|
|
||||||
"a reference to an operand wrapped in a choice": {
|
|
||||||
map[string]string{
|
|
||||||
"cat": `{"format":"{calc(n * 2, 2)} {q}","n":{"format":"{v}","v":["1","2"]},` +
|
|
||||||
`"q":"{/hop}"}`,
|
|
||||||
"hop": `"{/cat.n}"`,
|
|
||||||
},
|
|
||||||
`{q} renders "n"`,
|
|
||||||
},
|
|
||||||
// The reaching route can be another operand, which is not a token — so the
|
|
||||||
// error must name it as one rather than invent a {b} the format never wrote.
|
|
||||||
"an operand reaching another operand": {
|
|
||||||
map[string]string{
|
|
||||||
"cat": `{"format":"{calc(a + b, 0)}","a":{"format":"{x}","x":["1","2"]},` +
|
|
||||||
`"b":"{/hop}"}`,
|
|
||||||
"hop": `"{/cat.a}"`,
|
|
||||||
},
|
|
||||||
`calc operand "b" renders "a"`,
|
|
||||||
},
|
|
||||||
// Reaching *into* the operand is the same shape as naming it: the draw the
|
|
||||||
// calc holds settled that value too. The token spelling {net.v} is already
|
|
||||||
// rejected, so the reference spelling has to be.
|
|
||||||
"a reference into the operand": {
|
|
||||||
map[string]string{
|
|
||||||
"cat": `{"format":"{calc(net * 2, 2)}|{/hop}",` +
|
|
||||||
`"net":{"format":"{v}","v":["10.00","20.00"]}}`,
|
|
||||||
"hop": `"{/cat.net.v}"`,
|
|
||||||
},
|
|
||||||
`{/hop} renders "net"`,
|
|
||||||
},
|
|
||||||
"a reference into the operand one level down": {
|
|
||||||
map[string]string{
|
|
||||||
"cat": `{"format":"{calc(net * 2, 2)}|{q}","net":{"format":"{v}","v":["10.00","20.00"]},` +
|
|
||||||
`"q":"{/hop}"}`,
|
|
||||||
"hop": `"{/cat.net.v}"`,
|
|
||||||
},
|
|
||||||
`{q} renders "net"`,
|
|
||||||
},
|
|
||||||
// Each head is walked against its own seen set. Sharing one across heads
|
|
||||||
// would let the first head's walk mark the only route to the second's draw,
|
|
||||||
// and this violation sits on the later head in sorted order.
|
|
||||||
"a violation on the second of two held heads": {
|
|
||||||
map[string]string{
|
|
||||||
"cat": `{"format":"{calc(a + b, 0)} {w}","a":{"format":"{x}","x":["1","2"]},` +
|
|
||||||
`"b":{"format":"{y}","y":["3","4"]},"w":"{/hop}"}`,
|
|
||||||
"hop": `"{/cat.b}"`,
|
|
||||||
},
|
|
||||||
`{w} renders "b"`,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for name, c := range rejected {
|
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files)))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "a {calc()} also reads") {
|
|
||||||
t.Errorf("%s: New = %v, want the second route to the operand rejected", name, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), c.want) {
|
|
||||||
t.Errorf("%s: New = %v, want it to name %q", name, err, c.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
accepted := map[string]map[string]string{
|
|
||||||
// The spellings that read the one draw: the operand itself, and a bare token.
|
|
||||||
"only the operand and a bare token": {
|
|
||||||
"cat": `{"format":"{net} x 2 = {calc(net * 2, 2)}","net":["10.00","20.00"]}`,
|
|
||||||
},
|
|
||||||
// A sibling the operand's format never renders is no part of its value. A
|
|
||||||
// path head differs — a path may read into anything the level contains —
|
|
||||||
// which is why that half of the fence covers containment.
|
|
||||||
"a reference to a sibling the operand never renders": {
|
|
||||||
"cat": `{"format":"{calc(net * 2, 2)} {unit}",` +
|
|
||||||
`"net":{"format":"{v}","v":["1","2"],"spare":["kg","lb"]},` +
|
|
||||||
`"unit":"{/hop}"}`,
|
|
||||||
"hop": `"{/cat.net.spare}"`,
|
|
||||||
},
|
|
||||||
// Two operands drawing from one source are two names, so two draws: each is
|
|
||||||
// held under its own name and shown once, and neither can disagree.
|
|
||||||
"two operands sharing one source": {
|
|
||||||
"die": `["1","2","3","4","5","6"]`,
|
|
||||||
"cat": `{"format":"{d1} + {d2} = {calc(d1 + d2, 0)}","d1":"{/die}",` +
|
|
||||||
`"d2":"{/die}"}`,
|
|
||||||
},
|
|
||||||
// Likewise a reference drawing from what the operand draws from: {/common}
|
|
||||||
// and net are two names, not two spellings of one field.
|
|
||||||
"a reference to what an operand renders through": {
|
|
||||||
"common": `["1","2"]`,
|
|
||||||
"cat": `{"format":"{calc(net * 2, 2)} {/common}","net":"{/common}"}`,
|
|
||||||
},
|
|
||||||
// A fixed string cannot disagree with itself, so it needs no fence.
|
|
||||||
"a literal operand named twice": {
|
|
||||||
"cat": `{"format":"{calc(n * 2, 0)} {/hop}","n":"5"}`,
|
|
||||||
"hop": `"{/cat.n}"`,
|
|
||||||
},
|
|
||||||
// One node reached twice while walking the operand: the walk must not
|
|
||||||
// revisit it, and the repeat is not a second route to anything.
|
|
||||||
"an operand that renders one field twice": {
|
|
||||||
"cat": `{"format":"{calc(v * 2, 0)}","v":{"format":"{a}{a}","a":["1","2"]}}`,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for name, files := range accepted {
|
|
||||||
if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil {
|
|
||||||
t.Errorf("%s: New = %v, want it accepted", name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAPathReachesEveryVariantItMightDraw(t *testing.T) {
|
|
||||||
// A path through a choice may land in any variant, so both walks have to see
|
|
||||||
// all of them. Reaching only the first leaves whatever hides in a later
|
|
||||||
// variant to be found at render — a cycle there is fatal, and a second route
|
|
||||||
// to a held level disagrees silently.
|
|
||||||
rejected := map[string]struct {
|
|
||||||
files map[string]string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
"a cycle in a later variant": {
|
|
||||||
map[string]string{
|
|
||||||
"cat": `{"format":"{p.x}","p":[{"format":"h","x":"safe"},{"format":"h","x":"{/hop}"}]}`,
|
|
||||||
"hop": `"{/cat}"`,
|
|
||||||
},
|
|
||||||
"reference cycle",
|
|
||||||
},
|
|
||||||
"a second route in a later variant": {
|
|
||||||
map[string]string{
|
|
||||||
"cat": `{"format":"{p.x} {q.y}","p":[{"format":"h","x":"safe"},{"format":"h","x":"{/hop}"}],` +
|
|
||||||
`"q":{"format":"{y}","y":["1","2"]}}`,
|
|
||||||
"hop": `"{/cat.q}"`,
|
|
||||||
},
|
|
||||||
"reads a path into",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for name, c := range rejected {
|
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, c.files)))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), c.want) {
|
|
||||||
t.Errorf("%s: New = %v, want it to mention %q", name, err, c.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestALevelAPathNeverRendersIsAccepted(t *testing.T) {
|
|
||||||
// A path token does not expand its head's format, so a reference sitting in
|
|
||||||
// that format is not a second route to anything: it is never rendered by the
|
|
||||||
// path at all. Both orders must load. p and q sit one level down, so they are
|
|
||||||
// one column of thing rather than two that would read one another.
|
|
||||||
accepted := map[string]map[string]string{
|
|
||||||
"reference in the head's own format": {
|
|
||||||
"thing": `{"format":"{inner}","inner":{"format":"{p.first} {q.a}",` +
|
|
||||||
`"p":{"format":"{first} {/hop}","first":["A","B"]},"q":{"format":"{a}","a":["1","2"]}}}`,
|
|
||||||
"hop": `"{/thing.inner.q}"`,
|
|
||||||
},
|
|
||||||
"the mirror shape": {
|
|
||||||
"thing": `{"format":"{inner}","inner":{"format":"{p.first} {q.a}",` +
|
|
||||||
`"p":{"format":"{first}","first":["A","B"]},"q":{"format":"{a} {/hop}","a":["1","2"]}}}`,
|
|
||||||
"hop": `"{/thing.inner.p}"`,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for name, files := range accepted {
|
|
||||||
if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil {
|
|
||||||
t.Errorf("%s: New = %v, want it accepted", name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestADeepDiamondChainLoads(t *testing.T) {
|
|
||||||
// A diamond chain is 2^n routes through n nodes. The search past a bound level
|
|
||||||
// must walk each node once, not once per route, or a deep chain never loads.
|
|
||||||
files := map[string]string{"l0": `"x"`}
|
|
||||||
for i := 1; i <= 30; i++ {
|
|
||||||
files[fmt.Sprintf("l%d", i)] = fmt.Sprintf(
|
|
||||||
`{"format":"{a}{b}","a":"{/l%d}","b":"{/l%d}"}`, i-1, i-1)
|
|
||||||
}
|
|
||||||
files["thing"] = `{"format":"{p.first} {/l30}","p":{"format":"x","first":["A","B"]}}`
|
|
||||||
if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, files))); err != nil {
|
|
||||||
t.Fatalf("New = %v, want a deep diamond chain to load", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestASharedNodeIsWalkedOnce(t *testing.T) {
|
|
||||||
// Two fields reaching one node make the render graph a diamond, not a tree.
|
|
||||||
// The search past a bound level must take that in its stride rather than walk
|
|
||||||
// the shared node once per route.
|
|
||||||
f, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
|
|
||||||
"cat": `{"format":"{p.first}|{q}","p":{"format":"{first}","first":["Anna","Bo"]},` +
|
|
||||||
`"q":{"format":"{a}{b}","a":"{/shared}","b":"{/shared}"}}`,
|
|
||||||
"shared": `"x"`,
|
|
||||||
})), WithSeed(1))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New = %v, want a shared node accepted", err)
|
|
||||||
}
|
|
||||||
if got, err := f.Fake("cat"); err != nil || !strings.HasSuffix(got, "|xx") {
|
|
||||||
t.Fatalf("Fake(cat) = %q, %v, want it to end in xx", got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheEarlierReaderIsNamed(t *testing.T) {
|
|
||||||
// A token and a calc operand can name one level. The one the format writes
|
|
||||||
// first is the one reported, so the error points at the same place a reader
|
|
||||||
// looking at the format would start.
|
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
|
|
||||||
"cat": `{"format":"{calc(p * 1)} {p} {p.first}","p":{"format":"{first}","first":["1","2"]}}`,
|
|
||||||
})))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), `calc operand "p"`) {
|
|
||||||
t.Fatalf("New = %v, want the calc operand named, being written first", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReferenceToAMatchingStringIsAccepted(t *testing.T) {
|
|
||||||
// A literal renders one fixed string, so no draw of it can disagree with a
|
|
||||||
// held one. Two unrelated literals that merely spell the same text must not
|
|
||||||
// read as the same node — the trap when comparing a value type.
|
|
||||||
if _, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
|
|
||||||
"cat": `{"format":"{p.city} {/other.tag}","p":{"format":"{city}","city":"Stockholm"}}`,
|
|
||||||
"other": `{"format":"x","tag":"Stockholm"}`,
|
|
||||||
}))); err != nil {
|
|
||||||
t.Fatalf("New = %v, want a reference to a matching string accepted", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNestedChoiceDrawsOneVariant(t *testing.T) {
|
|
||||||
// A choice item may itself be a choice, so drawing a bound head unwraps until
|
|
||||||
// it reaches a value. Stopping at one level leaves a choice where the walk
|
|
||||||
// expects a template.
|
|
||||||
f := engine(21)
|
|
||||||
seen := map[string]bool{}
|
|
||||||
for i := 0; i < 200; i++ {
|
|
||||||
seen[mustRender(t, f, `{"format":"{p.x}","p":[{"format":"{x}","x":"1"},{"format":"{x}","x":"2"}]}`)] = true
|
|
||||||
}
|
|
||||||
if !seen["1"] || !seen["2"] || len(seen) != 2 {
|
|
||||||
t.Fatalf("200 draws produced %v, want 1 and 2", seen)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRepeatingLevelIsNamedInTheError(t *testing.T) {
|
|
||||||
// The level carrying the repeat is the one to fix, so the error names it
|
|
||||||
// rather than the head the path started from.
|
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
|
|
||||||
"cat": `{"format":"[{p.a.b}]","p":{"format":"{a}","a":{"format":"{b}","repeat":3,"separator":",","b":"z"}}}`,
|
|
||||||
})))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), `"p.a"`) {
|
|
||||||
t.Fatalf("New = %v, want it to name the level p.a that carries the repeat", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPathIntoARepeatingLevelBehindAChoiceIsRejected(t *testing.T) {
|
|
||||||
// A choice of rows is the shape this feature is for, so the repeat rule has to
|
|
||||||
// reach inside one — otherwise the direct spelling is a load error and the same
|
|
||||||
// mistake behind a choice silently drops the repeat.
|
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
|
|
||||||
"cat": `{"format":"[{p.a}]","p":[{"format":"{a}","repeat":3,"separator":",","a":"x"},{"format":"{a}","a":"y"}]}`,
|
|
||||||
})))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "repeat") {
|
|
||||||
t.Fatalf("New = %v, want the repeat behind a choice rejected", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPathIntoARepeatingLevelIsRejected(t *testing.T) {
|
|
||||||
// A path reads one level's draw, so it can never apply that level's repeat.
|
|
||||||
// The engine rejects options that cannot take effect, and this is one.
|
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
|
|
||||||
"cat": `{"format":"[{p.a}]","p":{"format":"{a}","repeat":3,"separator":",","a":"z"}}`,
|
|
||||||
})))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "repeat") {
|
|
||||||
t.Fatalf("New = %v, want a path into a repeating level rejected", err)
|
|
||||||
}
|
|
||||||
_, err = New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
|
|
||||||
"cat": `{"format":"[{p.a}]","p":{"format":"{a}","a":"{/word.w}","drawGroup":"g"}}`,
|
|
||||||
"word": `{"format":"{w}","w":["x","y"]}`,
|
|
||||||
})))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), `the level "p" carries a drawGroup`) {
|
|
||||||
t.Fatalf("New = %v, want a path into a level carrying a group rejected", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPathIntoAPlainTemplateNamesTheMissingField(t *testing.T) {
|
|
||||||
// A head that is one template, not a choice, reports the missing segment
|
|
||||||
// directly — there are no variants to compare.
|
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
|
|
||||||
"cat": `{"format":"{a.nope}","a":{"format":"x","b":"1"}}`,
|
|
||||||
})))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), `no field "nope"`) {
|
|
||||||
t.Fatalf("New = %v, want it to name the missing field", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFieldWithoutAPathTokenStillDrawsEachTime(t *testing.T) {
|
|
||||||
// A format with no dotted token keeps today's behaviour: every {word} is an
|
|
||||||
// independent draw.
|
|
||||||
f := engine(3)
|
|
||||||
varied := false
|
|
||||||
for i := 0; i < 200 && !varied; i++ {
|
|
||||||
got := mustRender(t, f, `{"format":"{w}{w}{w}","w":["a","b","c","d","e"]}`)
|
|
||||||
varied = got[0] != got[1] || got[1] != got[2]
|
|
||||||
}
|
|
||||||
if !varied {
|
|
||||||
t.Fatal("200 draws of {w}{w}{w} never varied, want independent draws")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTwoBoundHeadsDrawIndependently(t *testing.T) {
|
|
||||||
// Binding is per head, so two correlated sets in one format compose instead
|
|
||||||
// of needing a cross product of variants.
|
|
||||||
f := engine(4)
|
|
||||||
seen := map[string]bool{}
|
|
||||||
for i := 0; i < 400; i++ {
|
|
||||||
seen[mustRender(t, f, `{"format":"{p.v}{q.v}",
|
|
||||||
"p":[{"format":"{v}","v":"A"},{"format":"{v}","v":"B"}],
|
|
||||||
"q":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}]}`)] = true
|
|
||||||
}
|
|
||||||
for _, want := range []string{"A1", "A2", "B1", "B2"} {
|
|
||||||
if !seen[want] {
|
|
||||||
t.Fatalf("400 draws produced %v, want every combination including %q", seen, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRepeatRedrawsTheBinding(t *testing.T) {
|
|
||||||
// The binding lives for one expansion, so each repetition is a fresh draw —
|
|
||||||
// and each repetition is internally consistent.
|
|
||||||
f := engine(5)
|
|
||||||
tmpl := strings.Replace(places("{place.postal-code} {place.locality}"), `"format":"{place.postal-code} {place.locality}"`,
|
|
||||||
`"format":"{place.postal-code} {place.locality}","repeat":6,"separator":"\n"`, 1)
|
|
||||||
varied := false
|
|
||||||
for i := 0; i < 40; i++ {
|
|
||||||
got := mustRender(t, f, tmpl)
|
|
||||||
lines := strings.Split(got, "\n")
|
|
||||||
if len(lines) != 6 {
|
|
||||||
t.Fatalf("repeat 6 produced %d lines", len(lines))
|
|
||||||
}
|
|
||||||
for _, line := range lines {
|
|
||||||
if !agree.MatchString(line) {
|
|
||||||
t.Fatalf("repetition %q does not pair", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if lines[0] != lines[1] || lines[1] != lines[2] {
|
|
||||||
varied = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !varied {
|
|
||||||
t.Fatal("40 renders of repeat 6 never varied within a render, want a redraw per repetition")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNestedTemplateKeepsItsOwnBinding(t *testing.T) {
|
|
||||||
// A binding belongs to the expansion that made it: an inner template's own
|
|
||||||
// place is not the outer one's.
|
|
||||||
f := engine(6)
|
|
||||||
tmpl := `{"format":"{place.v}{inner}",
|
|
||||||
"place":[{"format":"{v}","v":"A"},{"format":"{v}","v":"B"}],
|
|
||||||
"inner":{"format":"{place.v}","place":[{"format":"{v}","v":"A"},{"format":"{v}","v":"B"}]}}`
|
|
||||||
seen := map[string]bool{}
|
|
||||||
for i := 0; i < 400; i++ {
|
|
||||||
seen[mustRender(t, f, tmpl)] = true
|
|
||||||
}
|
|
||||||
if !seen["AB"] && !seen["BA"] {
|
|
||||||
t.Fatalf("400 draws produced %v, want the inner binding to be independent", seen)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBoundDrawIsSeedStable(t *testing.T) {
|
|
||||||
tmpl := places("{place.postal-code} {place.locality}")
|
|
||||||
a := mustRender(t, engine(42), tmpl)
|
|
||||||
b := mustRender(t, engine(42), tmpl)
|
|
||||||
if a != b {
|
|
||||||
t.Fatalf("same seed gave %q and %q, want an identical draw", a, b)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDottedTokenErrors(t *testing.T) {
|
|
||||||
// A path token is validated where every other token is: at New, naming what
|
|
||||||
// is wrong, so a typo can never reach a render.
|
|
||||||
rejected := map[string]struct {
|
|
||||||
file string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
"unknown head": {
|
|
||||||
`{"format":"{nope.x}","place":{"format":"{v}","v":"A"}}`,
|
|
||||||
`no field "nope"`,
|
|
||||||
},
|
|
||||||
"unknown tail": {
|
|
||||||
`{"format":"{place.nope}","place":[{"format":"{v}","v":"A"},{"format":"{v}","v":"B"}]}`,
|
|
||||||
`"nope"`,
|
|
||||||
},
|
|
||||||
"tail only some variants carry": {
|
|
||||||
`{"format":"{place.locality}","place":[{"format":"{locality}","locality":"A"},"x"]}`,
|
|
||||||
`locality`,
|
|
||||||
},
|
|
||||||
"path into a literal": {
|
|
||||||
`{"format":"{x.y}","x":"plain"}`,
|
|
||||||
`"y"`,
|
|
||||||
},
|
|
||||||
"dotted field key": {
|
|
||||||
`{"format":"[{a.b}]","a.b":"V"}`,
|
|
||||||
`contains "."`,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for name, c := range rejected {
|
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": c.file})))
|
|
||||||
if err == nil {
|
|
||||||
t.Errorf("%s: New = nil error, want it rejected at load", name)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), c.want) {
|
|
||||||
t.Errorf("%s: New = %v, want it to mention %q", name, err, c.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSamePathReadTwiceReadsOneValue(t *testing.T) {
|
|
||||||
// A path is drawn once per expansion, so reading it twice reads one value —
|
|
||||||
// what a name shown in a display form and again in an address needs.
|
|
||||||
f := engine(7)
|
|
||||||
for i := 0; i < 300; i++ {
|
|
||||||
got := mustRender(t, f, `{"format":"{p.first}|{p.first}","p":{"format":"{first}","first":["Anna","Astrid","Elin","Karin"]}}`)
|
|
||||||
parts := strings.Split(got, "|")
|
|
||||||
if parts[0] != parts[1] {
|
|
||||||
t.Fatalf("draw %d = %q, want one value read twice", i, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDifferentTailsUnderOneHeadShareTheRow(t *testing.T) {
|
|
||||||
// Two tails of one head stay in the same variant, each keeping its own value.
|
|
||||||
f := engine(8)
|
|
||||||
for i := 0; i < 300; i++ {
|
|
||||||
got := mustRender(t, f, `{"format":"{p.a}{p.b}",
|
|
||||||
"p":[{"format":"x","a":"A","b":"1"},{"format":"y","a":"B","b":"2"}]}`)
|
|
||||||
if got != "A1" && got != "B2" {
|
|
||||||
t.Fatalf("draw %d = %q, want a row's own pair", i, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// deepPlaces nests the correlated facts a level down: the row holds an addr, and
|
|
||||||
// the addr holds the city and the zip that belongs to it.
|
|
||||||
const deepPlaces = `{"format":"%s","p":{"format":"{addr}","addr":[{"format":"{city}","city":"Stockholm","zip":"11111"},{"format":"{city}","city":"Kiruna","zip":"98100"}]}}`
|
|
||||||
|
|
||||||
func TestPathHoldsItsDrawAtEveryLevel(t *testing.T) {
|
|
||||||
// A choice below the head is drawn once too, so two paths sharing a prefix
|
|
||||||
// share every level of it — not just the row they started from.
|
|
||||||
f := engine(11)
|
|
||||||
seen := map[string]bool{}
|
|
||||||
for i := 0; i < 500; i++ {
|
|
||||||
got := mustRender(t, f, strings.Replace(deepPlaces, "%s", "{p.addr.city}/{p.addr.zip}", 1))
|
|
||||||
if got != "Stockholm/11111" && got != "Kiruna/98100" {
|
|
||||||
t.Fatalf("draw %d = %q, want a city and the zip that belongs to it", i, got)
|
|
||||||
}
|
|
||||||
seen[got] = true
|
|
||||||
}
|
|
||||||
if len(seen) < 2 {
|
|
||||||
t.Fatalf("500 draws produced only %v, want both", seen)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPathHoldsItsDrawThreeLevelsDown(t *testing.T) {
|
|
||||||
// The rule does not run out at two: every level a path passes through is held.
|
|
||||||
f := engine(12)
|
|
||||||
tmpl := `{"format":"{p.geo.town.name}/{p.geo.town.zip} {p.geo.region}","p":{"format":"{geo}","geo":[{"format":"{town}","region":"Norrbotten","town":[{"format":"{name}","name":"Kiruna","zip":"98100"},{"format":"{name}","name":"Luleå","zip":"97200"}]},{"format":"{town}","region":"Skåne","town":[{"format":"{name}","name":"Malmö","zip":"21100"},{"format":"{name}","name":"Lund","zip":"22100"}]}]}}`
|
|
||||||
want := map[string]bool{
|
|
||||||
"Kiruna/98100 Norrbotten": true, "Luleå/97200 Norrbotten": true,
|
|
||||||
"Malmö/21100 Skåne": true, "Lund/22100 Skåne": true,
|
|
||||||
}
|
|
||||||
seen := map[string]bool{}
|
|
||||||
for i := 0; i < 600; i++ {
|
|
||||||
got := mustRender(t, f, tmpl)
|
|
||||||
if !want[got] {
|
|
||||||
t.Fatalf("draw %d = %q, want a town with its own zip and region", i, got)
|
|
||||||
}
|
|
||||||
seen[got] = true
|
|
||||||
}
|
|
||||||
if len(seen) != len(want) {
|
|
||||||
t.Fatalf("600 draws produced %v, want all four towns", seen)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPathWalksVariantsOfDifferentShape(t *testing.T) {
|
|
||||||
// One variant holds addr as a plain template where another holds a choice.
|
|
||||||
// Both carry the path, so both must walk — the walk reads what it drew, never
|
|
||||||
// assuming a shape.
|
|
||||||
f := engine(13)
|
|
||||||
tmpl := `{"format":"{p.addr.city}/{p.addr.zip}","p":[
|
|
||||||
{"format":"{addr}","addr":{"format":"{city}","city":"Kiruna","zip":"98100"}},
|
|
||||||
{"format":"{addr}","addr":[
|
|
||||||
{"format":"{city}","city":"Malmö","zip":"21100"},
|
|
||||||
{"format":"{city}","city":"Lund","zip":"22100"}]}]}`
|
|
||||||
for i := 0; i < 400; i++ {
|
|
||||||
got := mustRender(t, f, tmpl)
|
|
||||||
if got != "Kiruna/98100" && got != "Malmö/21100" && got != "Lund/22100" {
|
|
||||||
t.Fatalf("draw %d = %q, want a city with its own zip", i, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDeepPathsUnderOneHeadStayIndependentWhereTheyDiverge(t *testing.T) {
|
|
||||||
// Two paths share only what they actually share: a common prefix is one draw,
|
|
||||||
// and each keeps its own draw past the point they part.
|
|
||||||
f := engine(14)
|
|
||||||
tmpl := `{"format":"{p.a.v}{p.b.v}","p":{"format":"x","a":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}],"b":[{"format":"{v}","v":"1"},{"format":"{v}","v":"2"}]}}`
|
|
||||||
seen := map[string]bool{}
|
|
||||||
for i := 0; i < 400; i++ {
|
|
||||||
seen[mustRender(t, f, tmpl)] = true
|
|
||||||
}
|
|
||||||
for _, want := range []string{"11", "12", "21", "22"} {
|
|
||||||
if !seen[want] {
|
|
||||||
t.Fatalf("400 draws produced %v, want every combination including %q", seen, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEmptyPathSegmentIsRejected(t *testing.T) {
|
|
||||||
// "{a.}" and "{a..b}" are unfinished paths, and the segment naming nothing is
|
|
||||||
// reported as that rather than as a missing field. An empty name is rejected
|
|
||||||
// where it is authored, so no data can make these resolve.
|
|
||||||
rejected := map[string]string{
|
|
||||||
"trailing dot": `{"format":"[{a.}]","a":"x"}`,
|
|
||||||
"double dot": `{"format":"[{a..b}]","a":"x"}`,
|
|
||||||
}
|
|
||||||
for name, file := range rejected {
|
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{"cat": file})))
|
|
||||||
if err == nil {
|
|
||||||
t.Errorf("%s: New = nil error, want the unfinished path rejected", name)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "empty segment") {
|
|
||||||
t.Errorf("%s: New = %v, want it to name the empty segment", name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCycleThroughAPathTokenIsRejected(t *testing.T) {
|
|
||||||
// A path token is a render edge like any other, so a cycle routed through one
|
|
||||||
// must be caught at New. Reaching render would be fatal: the recursion never
|
|
||||||
// terminates, and a stack overflow cannot be recovered.
|
|
||||||
_, err := New(WithoutShippedData(), WithDataPath(writeData(t, map[string]string{
|
|
||||||
"a": `{"format":"{p.x}","p":{"format":"{x}","x":"{/b}"}}`,
|
|
||||||
"b": `"{/a}"`,
|
|
||||||
})))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "reference cycle") {
|
|
||||||
t.Fatalf("New = %v, want the cycle through {p.x} rejected", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBoundPathIsReachableByFake(t *testing.T) {
|
|
||||||
// Binding changes how a format reads a sibling, not what List and Fake offer:
|
|
||||||
// the sub-fields stay addressable on their own.
|
|
||||||
dir := writeData(t, map[string]string{"address": places("{place.postal-code} {place.locality}")})
|
|
||||||
f, err := New(WithoutShippedData(), WithDataPath(dir), WithSeed(9))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New = %v", err)
|
|
||||||
}
|
|
||||||
for _, path := range []string{"address", "address.place", "address.place.locality", "address.place.postal-code"} {
|
|
||||||
if _, err := f.Fake(path); err != nil {
|
|
||||||
t.Errorf("Fake(%q) = %v, want it to render", path, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRepeatedBareTokenOfAHeldNameIsRejected(t *testing.T) {
|
|
||||||
for src, want := range map[string]string{
|
|
||||||
`{"format":"{w} {w} {uppercase(w)}","w":["a","b"]}`: "write {w} once",
|
|
||||||
`{"format":"{w} {w|x} {uppercase(w)}","w":["a","b"],"x":["c","d"]}`: "write {w} once",
|
|
||||||
`{"format":"{n} + {n} = {calc(n * 2)}","n":["1","2"]}`: "write {n} once",
|
|
||||||
`{"format":"{p.a} {q} {q} {lowercase(q)}","p":{"format":"{a}","a":"1"},"q":["A","B"]}`: "write {q} once",
|
|
||||||
} {
|
|
||||||
_, err := compile(parse(t, src))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "holds") {
|
|
||||||
t.Errorf("compile(%s) = %v, want the repeated token rejected naming %s", src, err, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, ok := range []string{
|
|
||||||
`{"format":"{w} {w}","w":["a","b"]}`,
|
|
||||||
`{"format":"{w} {uppercase(w)}","w":["a","b"]}`,
|
|
||||||
`{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99","5.00"],"qty":["3","7"]}`,
|
|
||||||
`{"format":"{uppercase(w)} {uppercase(w)}","w":["a","b"]}`,
|
|
||||||
} {
|
|
||||||
if _, err := compile(parse(t, ok)); err != nil {
|
|
||||||
t.Errorf("compile(%s) = %v, want it accepted", ok, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadFieldPanicsOnAnUnheldPath(t *testing.T) {
|
|
||||||
tm, ok := compiled(t, `{"format":"{w}","w":{"format":"{x}","x":"1"}}`).(*template)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("not a template")
|
|
||||||
}
|
|
||||||
mustPanic(t, "unheld arm with a path", func() {
|
|
||||||
readField(engine(1).rand, tm, nil, drawScope{}, arm{name: "w.x", key: "w", tail: []string{"x"}, path: "w.x"})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
+120
@@ -0,0 +1,120 @@
|
|||||||
|
package fejkdata
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"go/build"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestPackageImports holds each package in the module to the module packages it may import.
|
||||||
|
func TestPackageImports(t *testing.T) {
|
||||||
|
mod, err := os.ReadFile("go.mod")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
module := ""
|
||||||
|
for _, line := range strings.Split(string(mod), "\n") {
|
||||||
|
if f := strings.Fields(line); len(f) > 1 && f[0] == "module" {
|
||||||
|
module = strings.Trim(f[1], `"`)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if module == "" {
|
||||||
|
t.Fatal("go.mod names no module")
|
||||||
|
}
|
||||||
|
checked := 0
|
||||||
|
allowed := map[string][]string{
|
||||||
|
".": {"internal/builtinfunc", "internal/datafiles", "internal/datatype", "internal/drawstate", "internal/grammar", "internal/invariant", "internal/jsonvalue", "internal/proven", "internal/rows"},
|
||||||
|
"cli": {"."},
|
||||||
|
"cmd/fejkdata": {"cli", "data"},
|
||||||
|
"data": {"data/en_US", "data/geo/SE", "data/geo/US", "data/misc", "data/sv_SE"},
|
||||||
|
"data/en_US": {"data/geo/US"},
|
||||||
|
"data/geo/SE": nil,
|
||||||
|
"data/geo/US": nil,
|
||||||
|
"data/misc": nil,
|
||||||
|
"data/sv_SE": {"data/geo/SE"},
|
||||||
|
"internal/builtinfunc": {"internal/datatype", "internal/drawstate", "internal/grammar", "internal/invariant", "internal/proven"},
|
||||||
|
"internal/datafiles": {"internal/grammar", "internal/jsonvalue"},
|
||||||
|
"internal/datatype": nil,
|
||||||
|
"internal/drawstate": nil,
|
||||||
|
"internal/grammar": nil,
|
||||||
|
"internal/invariant": nil,
|
||||||
|
"internal/jsonvalue": nil,
|
||||||
|
"internal/proven": {"internal/datatype", "internal/grammar", "internal/invariant"},
|
||||||
|
"internal/rows": {"internal/drawstate", "internal/grammar", "internal/invariant"},
|
||||||
|
}
|
||||||
|
testsAlso := map[string][]string{
|
||||||
|
".": {"data", "data/en_US", "data/geo/SE", "data/geo/US", "data/misc", "data/sv_SE"},
|
||||||
|
"cli": {"internal/grammar"},
|
||||||
|
"data/en_US": {"."},
|
||||||
|
"data/geo/SE": {"."},
|
||||||
|
"data/geo/US": {"."},
|
||||||
|
"data/misc": {"."},
|
||||||
|
"data/sv_SE": {"."},
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
eachPackage(t, func(dir string, pkg *build.Package) {
|
||||||
|
seen[dir] = true
|
||||||
|
may, listed := allowed[dir]
|
||||||
|
if !listed {
|
||||||
|
t.Errorf("package %s is not listed: add the module packages it may import", dir)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, imp := range slices.Concat(pkg.Imports, pkg.TestImports, pkg.XTestImports) {
|
||||||
|
may := may
|
||||||
|
if !slices.Contains(pkg.Imports, imp) {
|
||||||
|
may = slices.Concat(may, testsAlso[dir])
|
||||||
|
}
|
||||||
|
if imp != module && !strings.HasPrefix(imp, module+"/") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
checked++
|
||||||
|
rel := strings.TrimPrefix(strings.TrimPrefix(imp, module), "/")
|
||||||
|
if rel == "" {
|
||||||
|
rel = "."
|
||||||
|
}
|
||||||
|
if rel != dir && !slices.Contains(may, rel) {
|
||||||
|
t.Errorf("package %s imports %s, which its list does not allow", dir, rel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if checked == 0 {
|
||||||
|
t.Fatalf("no package imports a package of module %s, which go.mod names", module)
|
||||||
|
}
|
||||||
|
for dir := range allowed {
|
||||||
|
if !seen[dir] {
|
||||||
|
t.Errorf("listed package %s does not exist", dir)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// eachPackage calls fn with every Go package in the module and its slash-separated directory.
|
||||||
|
func eachPackage(t *testing.T, fn func(dir string, pkg *build.Package)) {
|
||||||
|
t.Helper()
|
||||||
|
err := filepath.WalkDir(".", func(dir string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil || !d.IsDir() {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if dir != "." && (strings.HasPrefix(d.Name(), ".") || d.Name() == "testdata") {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
pkg, err := build.ImportDir(dir, 0)
|
||||||
|
var noGo *build.NoGoError
|
||||||
|
if errors.As(err, &noGo) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fn(filepath.ToSlash(dir), pkg)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,188 @@
|
|||||||
|
package fejkdata
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata/internal/datafiles"
|
||||||
|
"github.com/larvit/fejkdata/internal/grammar"
|
||||||
|
)
|
||||||
|
|
||||||
|
// indexed is a category an index names and no call has reached yet, and the source holding it.
|
||||||
|
type indexed struct {
|
||||||
|
entry datafiles.IndexEntry
|
||||||
|
src datafiles.Source
|
||||||
|
}
|
||||||
|
|
||||||
|
// placeIndex sets every category the index names as unloaded in root, in place of what
|
||||||
|
// root holds at its path.
|
||||||
|
func placeIndex(root *folder, src datafiles.Source, index map[string]datafiles.IndexEntry) {
|
||||||
|
for _, p := range sortedNames(index) {
|
||||||
|
segs := strings.Split(p, ".")
|
||||||
|
madeFolder(root, segs[:len(segs)-1]).putUnloaded(segs[len(segs)-1], &indexed{entry: index[p], src: src})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// categoryAt is where a category, loaded or not, sits: the folder holding it, that
|
||||||
|
// folder's path, and its name.
|
||||||
|
type categoryAt struct {
|
||||||
|
dir []string
|
||||||
|
in *folder
|
||||||
|
name string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c categoryAt) unloaded() bool { return c.in.unloaded[c.name] != nil }
|
||||||
|
|
||||||
|
// categoryOn is the category, loaded or not, that a path from root names or passes.
|
||||||
|
func categoryOn(root *folder, segs []string) (categoryAt, bool) {
|
||||||
|
g, i := folderAt(root, segs)
|
||||||
|
if i == len(segs) {
|
||||||
|
return categoryAt{}, false
|
||||||
|
}
|
||||||
|
c := categoryAt{dir: segs[:i:i], in: g, name: segs[i]}
|
||||||
|
return c, c.unloaded() || g.children[c.name] != nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadCallerPath strips the / a caller's path may start with, splits the path, and loads the
|
||||||
|
// unloaded category it names or passes. The caller walks it next, and a walk loads nothing.
|
||||||
|
func (f *Generator) loadCallerPath(path string) (string, []string, error) {
|
||||||
|
path, err := grammar.CallerPath(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, fmt.Errorf("fejkdata: %w", err)
|
||||||
|
}
|
||||||
|
segs, err := grammar.SplitPath(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, fmt.Errorf("fejkdata: %w", err)
|
||||||
|
}
|
||||||
|
if c, ok := categoryOn(&f.root, segs); ok && c.unloaded() {
|
||||||
|
if err := loadReached(&f.root, nil, []categoryAt{c}); err != nil {
|
||||||
|
return "", nil, fmt.Errorf("fejkdata: %w", loadError{err})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return path, segs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// referenced is every category, loaded or not, that the templates of nodes reference, each
|
||||||
|
// reference read from the folder dir.
|
||||||
|
func referenced(root *folder, dir []string, nodes nodeSet) []categoryAt {
|
||||||
|
var out []categoryAt
|
||||||
|
_ = nodes(func(_ string, n node) error {
|
||||||
|
t, isTemplate := n.(*template)
|
||||||
|
if !isTemplate {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, name := range refTokens(t.tokens) {
|
||||||
|
segs, err := refSegments(name, dir)
|
||||||
|
if err != nil {
|
||||||
|
continue // resolveRefs reports it
|
||||||
|
}
|
||||||
|
if c, ok := categoryOn(root, segs); ok {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// needs is every category, loaded or not, that s references, the table beside s that its
|
||||||
|
// parent column names, and every unloaded table beside s whose parent s is.
|
||||||
|
func needs(root *folder, s categorySite) []categoryAt {
|
||||||
|
out := referenced(root, s.dir, siteNodes([]categorySite{s}))
|
||||||
|
if parent := parentOf(s.n); parent != "" {
|
||||||
|
out = append(out, categoryAt{dir: s.dir, in: s.in, name: parent})
|
||||||
|
}
|
||||||
|
for _, name := range sortedNames(s.in.unloaded) {
|
||||||
|
if s.in.unloaded[name].entry.Parent == s.name {
|
||||||
|
out = append(out, categoryAt{dir: s.dir, in: s.in, name: name})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadReached loads the unloaded categories among wanted and every unloaded category they
|
||||||
|
// or sites need, then runs sites and all it loaded through a whole load's pipeline. On an
|
||||||
|
// error it puts back what it loaded, so the next call fails the same way.
|
||||||
|
func loadReached(root *folder, sites []categorySite, wanted []categoryAt) error {
|
||||||
|
queue := wanted
|
||||||
|
for _, s := range sites {
|
||||||
|
queue = append(queue, needs(root, s)...)
|
||||||
|
}
|
||||||
|
var b batch
|
||||||
|
err := b.load(root, queue)
|
||||||
|
if err == nil {
|
||||||
|
err = b.run(root, sites)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
b.putBack()
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// batch is the categories one loadReached loaded, each beside the entry it was loaded from.
|
||||||
|
type batch []loaded
|
||||||
|
|
||||||
|
type loaded struct {
|
||||||
|
site categorySite
|
||||||
|
was *indexed
|
||||||
|
}
|
||||||
|
|
||||||
|
// load loads each unloaded category in queue and what it needs. It stops at a table whose
|
||||||
|
// parent column disagrees with its entry. Otherwise a table could load without its parent,
|
||||||
|
// or an earlier table could link to one that putBack then removes.
|
||||||
|
func (b *batch) load(root *folder, queue []categoryAt) error {
|
||||||
|
for ; len(queue) > 0; queue = queue[1:] {
|
||||||
|
c := queue[0]
|
||||||
|
x := c.in.unloaded[c.name]
|
||||||
|
if x == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := x.src.Load(c.dir, c.name, compileInto(func([]string) *folder { return c.in })); err != nil {
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
err = fmt.Errorf("%s indexes %s: %w", x.src.ManifestPath(), categoryPath(c.dir, c.name), err)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s := siteIn(c.dir, c.in, c.name)
|
||||||
|
*b = append(*b, loaded{site: s, was: x})
|
||||||
|
if parent := parentOf(s.n); parent != x.entry.Parent {
|
||||||
|
return fmt.Errorf("%s: the index entry for %s %s, and its table %s", x.src.ManifestPath(), s.path, namesParent(x.entry.Parent), namesParent(parent))
|
||||||
|
}
|
||||||
|
queue = append(queue, needs(root, s)...)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b batch) run(root *folder, sites []categorySite) error {
|
||||||
|
sites = slices.Clip(sites)
|
||||||
|
for _, l := range b {
|
||||||
|
sites = append(sites, l.site)
|
||||||
|
}
|
||||||
|
if len(sites) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return categoryPipeline(sites, root.children).run()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b batch) putBack() {
|
||||||
|
for _, l := range b {
|
||||||
|
l.site.in.putUnloaded(l.site.name, l.was)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func namesParent(parent string) string {
|
||||||
|
if parent == "" {
|
||||||
|
return "names no parent"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("names parent %q", parent)
|
||||||
|
}
|
||||||
|
|
||||||
|
func parentOf(n node) string {
|
||||||
|
if t, isTable := n.(*table); isTable {
|
||||||
|
return t.rows.Options().Parent
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
@@ -3,28 +3,30 @@ package fejkdata
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
|
||||||
|
"github.com/larvit/fejkdata/internal/grammar"
|
||||||
|
"github.com/larvit/fejkdata/internal/jsonvalue"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Template is an inline template compiled, referenced and validated against a
|
// Template is an inline template compiled, referenced and validated against a
|
||||||
// generator's loaded data once, ready to render many times with [Template.Fake].
|
// generator's loaded data once, ready to render many times with [Template.Fake].
|
||||||
// It is safe for concurrent use: Fake serializes on its generator's lock, so a
|
// It is safe for concurrent use: Fake serializes on its generator's lock, so a
|
||||||
// seeded sequence is reproducible only when a generator — and its templates — are
|
// sequence is reproducible only when one goroutine makes every draw from a generator
|
||||||
// drawn from one goroutine.
|
// and its templates.
|
||||||
type Template struct {
|
type Template struct {
|
||||||
g *Generator
|
g *Generator
|
||||||
n node
|
n node
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fake renders the template with one draw.
|
// Fake renders the template once; each call draws anew.
|
||||||
func (t *Template) Fake() string {
|
func (t *Template) Fake() string {
|
||||||
t.g.mu.Lock()
|
t.g.mu.Lock()
|
||||||
defer t.g.mu.Unlock()
|
defer t.g.mu.Unlock()
|
||||||
return renderOnce(t.g.rand, t.n)
|
return renderOnce(t.g.drawState, t.n)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewTemplate compiles an inline template — a format string or a JSON value — and
|
// NewTemplate compiles an inline template — a format string or a JSON value — and
|
||||||
// binds its references against the loaded tree, so repeated renders pay the
|
// resolves its references against the loaded tree, so repeated renders pay the
|
||||||
// compile and validation once. It shares [New]'s guarantees: a bad template errors
|
// compile and validation once. It shares [New]'s guarantees: a bad template errors
|
||||||
// here, and rendering cannot fail.
|
// here, and rendering cannot fail.
|
||||||
func (f *Generator) NewTemplate(input string) (*Template, error) {
|
func (f *Generator) NewTemplate(input string) (*Template, error) {
|
||||||
@@ -32,8 +34,10 @@ func (f *Generator) NewTemplate(input string) (*Template, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("fejkdata: %w", err)
|
return nil, fmt.Errorf("fejkdata: %w", err)
|
||||||
}
|
}
|
||||||
if err := bindInline(n, "template", f.categories, checkScope); err != nil {
|
f.mu.Lock()
|
||||||
return nil, fmt.Errorf("fejkdata: %w", err)
|
defer f.mu.Unlock()
|
||||||
|
if err := loadInline(&f.root, n, "template", false); err != nil {
|
||||||
|
return nil, f.noDataNote(fmt.Errorf("fejkdata: %w", err))
|
||||||
}
|
}
|
||||||
return &Template{g: f, n: n}, nil
|
return &Template{g: f, n: n}, nil
|
||||||
}
|
}
|
||||||
@@ -49,127 +53,70 @@ func (f *Generator) FakeTemplate(input string) (string, error) {
|
|||||||
return t.Fake(), nil
|
return t.Fake(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsTemplate reports whether arg is an inline template rather than a path, by its shape: a {
|
// IsTemplate reports whether arg is an inline template rather than a path, by its shape: an
|
||||||
// token, or a JSON object, array or string, is a template, and anything else is a path. A
|
// arg holding a {, or a valid JSON array or string, is a template, and anything else is a
|
||||||
// name never holds a bracket, a brace or a quote, so an arg holding one that is not valid
|
// path. A path errors where it holds a } or a quote, starts with [, or is no valid path. A path
|
||||||
// JSON names neither, and errors; so do a template of one reference alone, which is a path
|
// may start with /, as a reference does, and every call taking a path drops it.
|
||||||
// written as a template, and a path written with a leading /.
|
|
||||||
func IsTemplate(arg string) (bool, error) {
|
func IsTemplate(arg string) (bool, error) {
|
||||||
inline, err := isTemplate(arg)
|
inline, err := grammar.IsTemplate(arg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("fejkdata: %w", err)
|
return false, fmt.Errorf("fejkdata: %w", err)
|
||||||
}
|
}
|
||||||
return inline, nil
|
return inline, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func isTemplate(arg string) (bool, error) {
|
|
||||||
if strings.ContainsRune(arg, '{') || (isJSONStart(strings.TrimSpace(arg)) && json.Valid([]byte(arg))) {
|
|
||||||
if path, lone := loneReference(arg); lone {
|
|
||||||
return false, pathAdvice(path, fmt.Sprintf("%s is the path %s written as a template", arg, path))
|
|
||||||
}
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
if i := strings.IndexAny(arg, `}"`); i >= 0 {
|
|
||||||
return false, fmt.Errorf("%q holds a %q, which no path may, and it is not valid JSON, so it names no template either", arg, arg[i:i+1])
|
|
||||||
}
|
|
||||||
if strings.HasPrefix(strings.TrimSpace(arg), "[") {
|
|
||||||
return false, fmt.Errorf(`%q starts with "[" and is not valid JSON, so it names no template; a path starts with a name`, arg)
|
|
||||||
}
|
|
||||||
if _, err := splitPath(arg); err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
if path := strings.TrimLeft(arg, "/"); path != arg && path != "" {
|
|
||||||
return false, pathAdvice(path, fmt.Sprintf("path %s starts with /, and every path starts at the root already", arg))
|
|
||||||
}
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func isJSONStart(arg string) bool {
|
|
||||||
return strings.HasPrefix(arg, "[") || strings.HasPrefix(arg, `"`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// pathAdvice refuses a spelling of path, naming path to write, or why no name can spell it.
|
|
||||||
func pathAdvice(path, refusal string) error {
|
|
||||||
if err := checkPathNames(path); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return fmt.Errorf("%s; write %s", refusal, path)
|
|
||||||
}
|
|
||||||
|
|
||||||
// loneReference is the path a template spells when the value it holds — a format string, a
|
|
||||||
// JSON string, or an object holding only a format — is one reference token and nothing else.
|
|
||||||
func loneReference(arg string) (string, bool) {
|
|
||||||
var raw any
|
|
||||||
if json.Unmarshal([]byte(arg), &raw) != nil {
|
|
||||||
raw = arg
|
|
||||||
}
|
|
||||||
if m, isObject := raw.(map[string]any); isObject && len(m) == 1 {
|
|
||||||
raw = m["format"]
|
|
||||||
}
|
|
||||||
format, isString := raw.(string)
|
|
||||||
body, lone := loneRef(format)
|
|
||||||
if !isString || !lone {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
if strings.HasPrefix(body, "/") {
|
|
||||||
body = "/" + strings.TrimLeft(body, "/")
|
|
||||||
}
|
|
||||||
_, path, err := refShape(body)
|
|
||||||
return path, err == nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func compileInput(input string) (node, error) {
|
func compileInput(input string) (node, error) {
|
||||||
v, err := inputValue(input)
|
return compile(inputValue(input))
|
||||||
|
}
|
||||||
|
|
||||||
|
// inputValue reads an inline template as the value compile takes: the JSON object, array, string
|
||||||
|
// or null it holds, the whitespace around it dropped, or else the input itself as a format string,
|
||||||
|
// a bare number or boolean among them.
|
||||||
|
func inputValue(input string) jsonvalue.Value {
|
||||||
|
raw, err := jsonvalue.Decode([]byte(input))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return jsonvalue.String(input)
|
||||||
}
|
}
|
||||||
return compile(v)
|
switch raw.Any().(type) {
|
||||||
|
case json.Number, bool:
|
||||||
|
return jsonvalue.String(input)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
}
|
}
|
||||||
|
|
||||||
// inputValue reads an inline template as the value compile takes: the JSON value it holds, or
|
// loadInline loads the unloaded categories n reads, then runs n through the pipeline against root.
|
||||||
// the input itself as a format string when it is not JSON.
|
func loadInline(root *folder, n node, label string, typedByGo bool) error {
|
||||||
func inputValue(input string) (any, error) {
|
nodes := inlineNodes(n, label)
|
||||||
var raw any
|
if err := refuseFolderRefs(nodes); err != nil {
|
||||||
if err := json.Unmarshal([]byte(input), &raw); err != nil {
|
|
||||||
return input, nil
|
|
||||||
}
|
|
||||||
if trimmed := strings.TrimSpace(input); trimmed != input {
|
|
||||||
return nil, fmt.Errorf("a JSON template may not be padded with spaces, which a format string would render; write %s", trimmed)
|
|
||||||
}
|
|
||||||
return raw, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// bindInline links an inline node's references against root and runs check over it, naming its
|
|
||||||
// nodes from label.
|
|
||||||
func bindInline(n node, label string, root map[string]node, check func(nodeScope) error) error {
|
|
||||||
if t, isTemplate := n.(*template); isTemplate && t.drawGroup != "" {
|
|
||||||
return fmt.Errorf("%s: drawGroup %q names nothing, since nothing can reference an inline template; drop it", label, t.drawGroup)
|
|
||||||
}
|
|
||||||
scope := inlineScope(n, label)
|
|
||||||
if err := linkNodeRefs(scope, root); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return check(scope)
|
if err := loadReached(root, nil, referenced(root, nil, nodes)); err != nil {
|
||||||
|
return loadError{err}
|
||||||
|
}
|
||||||
|
return pipeline{
|
||||||
|
nodes: nodes,
|
||||||
|
resolve: func() error { return resolveInlineTemplates(nodes, root.children) },
|
||||||
|
typedByGo: typedByGo,
|
||||||
|
}.run()
|
||||||
}
|
}
|
||||||
|
|
||||||
// linkNodeRefs binds the references in an inline node's templates against the
|
// refuseFolderRefs refuses each reference in nodes not written from the root, {/x}: an
|
||||||
// loaded tree.
|
// inline node sits in no folder.
|
||||||
func linkNodeRefs(scope nodeScope, root map[string]node) error {
|
func refuseFolderRefs(nodes nodeSet) error {
|
||||||
return scope(func(path string, m node) error {
|
return nodes(func(label string, m node) error {
|
||||||
t, ok := m.(*template)
|
t, ok := m.(*template)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
t.keyDrawGroup("")
|
for _, name := range refTokens(t.tokens) {
|
||||||
for _, name := range refTokens(t.format) {
|
sigil, rest, err := grammar.RefShape(name)
|
||||||
sigil, rest, err := refShape(name)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%s: reference {%s}: %w", path, name, err)
|
return fmt.Errorf("%s: reference {%s}: %w", label, name, err)
|
||||||
}
|
}
|
||||||
if sigil != "/" {
|
if sigil != "/" {
|
||||||
return fmt.Errorf("%s: reference {%s}: an inline template has no folder; write {/%s}", path, name, rest)
|
return fmt.Errorf("%s: reference {%s}: an inline template has no folder; write {/%s}", label, name, rest)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return linkTemplateRefs(nil, path, "", t, root)
|
return nil
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
+44
-46
@@ -16,7 +16,7 @@ func tmpl(t *testing.T, f *Generator, input string) string {
|
|||||||
|
|
||||||
func shipped(t *testing.T, opts ...Option) *Generator {
|
func shipped(t *testing.T, opts ...Option) *Generator {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
f, err := New(append([]Option{WithSeed(1)}, opts...)...)
|
f, err := New(append([]Option{withShipped(), WithSeed(1)}, opts...)...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -53,13 +53,13 @@ func TestFakeTemplateJSONArray(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFakeTemplateCorrelatedReferences(t *testing.T) {
|
func TestFakeTemplateNamedReference(t *testing.T) {
|
||||||
dir := writeData(t, map[string]string{
|
dir := writeData(t, map[string]string{
|
||||||
"person": `[{"format":"{first} {last}","first":"Ada","last":"Lovelace"},{"format":"{first} {last}","first":"Bo","last":"Ek","born":"1990"}]`,
|
"person": `[{"format":"{first} {last}","first":"Ada","last":"Lovelace"},{"format":"{first} {last}","first":"Bo","last":"Ek","born":"1990"}]`,
|
||||||
})
|
})
|
||||||
f := newGenerator(t, dir, WithSeed(1))
|
f := newGenerator(t, dir, WithSeed(1))
|
||||||
for i := 0; i < 100; i++ {
|
for i := 0; i < 100; i++ {
|
||||||
got := tmpl(t, f, "{/person.first} {/person.last}")
|
got := tmpl(t, f, "{/person as p}{p.first} {p.last}")
|
||||||
if got != "Ada Lovelace" && got != "Bo Ek" {
|
if got != "Ada Lovelace" && got != "Bo Ek" {
|
||||||
t.Fatalf("correlated references = %q, want one person's first and last", got)
|
t.Fatalf("correlated references = %q, want one person's first and last", got)
|
||||||
}
|
}
|
||||||
@@ -104,13 +104,9 @@ func TestFakeTemplateErrors(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{`{"x":"Q"}`, "missing string \"format\""},
|
{`{"x":"Q"}`, "missing string \"format\""},
|
||||||
{`"{x}"`, `no field "x"`},
|
{`"{x}"`, `no field "x"`},
|
||||||
{`"{digits(0)}"`, "must be positive"},
|
{`"{digits(-1)}"`, "must not be negative"},
|
||||||
{`name: {/no.such.path}`, "no entry"},
|
{`name: {/no.such.path}`, "no entry"},
|
||||||
{`name: {..nope}`, "write {/nope}"},
|
{`name: {..nope}`, "write {/nope}"},
|
||||||
{`{"format":"x"}`, "is a string"},
|
|
||||||
{`{/misc.territory} {/misc.territory.alpha2}`, "renders a level that {/misc.territory.alpha2} reads a path into; name the fields you want instead, or draw them apart with a drawGroup"},
|
|
||||||
{`{"format":"{/misc.territory.alpha2} {x}","x":"{/misc.territory}"}`, "reads a path into"},
|
|
||||||
{`{"format":"{/misc.territory.alpha2}","drawGroup":"g"}`, "nothing can reference"},
|
|
||||||
} {
|
} {
|
||||||
_, err := f.FakeTemplate(c.input)
|
_, err := f.FakeTemplate(c.input)
|
||||||
if err == nil || !strings.Contains(err.Error(), c.want) {
|
if err == nil || !strings.Contains(err.Error(), c.want) {
|
||||||
@@ -127,12 +123,15 @@ func TestFakeTemplateJSONString(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPaddedJSONIsRejected(t *testing.T) {
|
func TestPaddedJSONIsTheJSON(t *testing.T) {
|
||||||
f := shipped(t)
|
f := shipped(t)
|
||||||
in := `{"format":"{x}","x":["a","b"]}`
|
in := `{"format":"{x}","x":["a","b"]}`
|
||||||
_, err := f.NewTemplate(" " + in + " ")
|
padded, err := f.NewTemplate(" \t" + in + "\n ")
|
||||||
if err == nil || !strings.Contains(err.Error(), "write "+in) {
|
if err != nil {
|
||||||
t.Fatalf("NewTemplate(padded JSON) = %v, want an error naming the unpadded spelling", err)
|
t.Fatalf("NewTemplate(padded JSON) = %v, want the JSON it pads", err)
|
||||||
|
}
|
||||||
|
if got := padded.Fake(); got != "a" && got != "b" {
|
||||||
|
t.Fatalf("padded JSON rendered %q, want a or b with no padding", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -153,46 +152,45 @@ func TestNewTemplateReusable(t *testing.T) {
|
|||||||
|
|
||||||
func TestIsTemplate(t *testing.T) {
|
func TestIsTemplate(t *testing.T) {
|
||||||
for arg, want := range map[string]bool{
|
for arg, want := range map[string]bool{
|
||||||
"sv_SE.person": false,
|
"sv_SE.person": false,
|
||||||
"person.last": false,
|
"person.last": false,
|
||||||
"name: {x}": true,
|
"name: {x}": true,
|
||||||
`{"format":"x"}`: true,
|
`{"format":"x"}`: true,
|
||||||
`["a","b"]`: true,
|
`["a","b"]`: true,
|
||||||
`[1, 2]`: true,
|
`[1, 2]`: true,
|
||||||
` ["a","b"]`: true, // padding is the template's own error, not a shape verdict
|
` ["a","b"]`: true, // padding is the template's own error, not a shape verdict
|
||||||
`"hello"`: true,
|
`"hello"`: true,
|
||||||
"{/a}{/b}": true,
|
"{/a}{/b}": true,
|
||||||
"{/a|/b}": true,
|
"{/a|/b}": true,
|
||||||
"{uppercase(/a)}": true,
|
"{uppercase(/a)}": true,
|
||||||
"{{/a}}": true,
|
"{{/a}}": true,
|
||||||
`"{/a} x"`: true,
|
`"{/a} x"`: true,
|
||||||
"x[1]": false, // a row selected by key or name
|
"x[1]": false, // a row selected by key or name
|
||||||
"x[St. Louis].y": false,
|
"x[St. Louis].y": false,
|
||||||
|
"{/sv_SE.person.last}": true,
|
||||||
|
`"{/sv_SE.person}"`: true,
|
||||||
|
"{.person.last}": true,
|
||||||
|
` "{/sv_SE.person}"`: true,
|
||||||
|
`{"format":"{/sv_SE.person}"}`: true,
|
||||||
|
"{//sv_SE.person}": true,
|
||||||
|
"{/misc.currency[US Dollar (Next day)]}": true,
|
||||||
|
"/sv_SE.person": false,
|
||||||
} {
|
} {
|
||||||
if got, err := IsTemplate(arg); err != nil || got != want {
|
if got, err := IsTemplate(arg); err != nil || got != want {
|
||||||
t.Errorf("IsTemplate(%q) = %v, %v; want %v", arg, got, err, want)
|
t.Errorf("IsTemplate(%q) = %v, %v; want %v", arg, got, err, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for arg, want := range map[string]string{
|
for arg, want := range map[string]string{
|
||||||
"[abc]": `starts with "["`,
|
"[abc]": `starts with "["`,
|
||||||
"[abc].field": `starts with "["`,
|
"[abc].field": `starts with "["`,
|
||||||
"x[1]y": `"]"`,
|
"x[1]y": `"]"`,
|
||||||
"x[[1]]": `"["`,
|
"x[[1]]": `"["`,
|
||||||
"x[]": "empty",
|
"x[]": "empty",
|
||||||
"a]b": `holds a "]"`,
|
"//sv_SE.person": "write /sv_SE.person",
|
||||||
"a}b": `holds a "}"`,
|
"a]b": `holds a "]"`,
|
||||||
`"abc`: `holds a "\""`,
|
"a}b": `holds a "}"`,
|
||||||
`"a]b`: `holds a "\""`, // the opener the reader typed, not the bracket behind it
|
`"abc`: `holds a "\""`,
|
||||||
"{/sv_SE.person.last}": "{/sv_SE.person.last} is the path sv_SE.person.last written as a template; write sv_SE.person.last",
|
`"a]b`: `holds a "\""`, // the opener the reader typed, not the bracket behind it
|
||||||
`"{/sv_SE.person}"`: "write sv_SE.person",
|
|
||||||
"{.person.last}": "write person.last",
|
|
||||||
"/sv_SE.person": "write sv_SE.person",
|
|
||||||
` "{/sv_SE.person}"`: "write sv_SE.person",
|
|
||||||
`{"format":"{/sv_SE.person}"}`: "write sv_SE.person",
|
|
||||||
"//sv_SE.person": "write sv_SE.person",
|
|
||||||
"{//sv_SE.person}": "{//sv_SE.person} is the path sv_SE.person written as a template; write sv_SE.person",
|
|
||||||
"/sv_SE/person": `path "sv_SE/person" contains "/"`,
|
|
||||||
"{/-}": `path "-" is reserved`,
|
|
||||||
} {
|
} {
|
||||||
if _, err := IsTemplate(arg); err == nil || !strings.Contains(err.Error(), want) {
|
if _, err := IsTemplate(arg); err == nil || !strings.Contains(err.Error(), want) {
|
||||||
t.Errorf("IsTemplate(%q) = %v; want it rejected naming %s", arg, err, want)
|
t.Errorf("IsTemplate(%q) = %v; want it rejected naming %s", arg, err, want)
|
||||||
|
|||||||
@@ -0,0 +1,119 @@
|
|||||||
|
// Package builtinfunc holds the {name(args)} functions a format calls: their checks, the
|
||||||
|
// draws they make, and the numbers their text is proven to read as.
|
||||||
|
package builtinfunc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata/internal/datatype"
|
||||||
|
"github.com/larvit/fejkdata/internal/drawstate"
|
||||||
|
"github.com/larvit/fejkdata/internal/proven"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Call is a builtin prepared for one call site: its args already parsed. It reads the
|
||||||
|
// output emitted so far in the current expansion (a derivation's payload) and the
|
||||||
|
// values of the operands it named, which the caller read for it.
|
||||||
|
type Call func(s *drawstate.State, emitted string, operands []string) string
|
||||||
|
|
||||||
|
// builtin is a format-string function invoked as {name(args)}. Its Call stays pure over its
|
||||||
|
// inputs, so seeded output is reproducible. arity is the exact arg count, or -1 for variadic,
|
||||||
|
// where checkArgs does all the validation.
|
||||||
|
type builtin struct {
|
||||||
|
arity int
|
||||||
|
// prep parses validated args once, at compile time, into the closure a render calls.
|
||||||
|
prep func(args []string) Call
|
||||||
|
checkArgs func(args []string) error
|
||||||
|
// operands names the fields the call reads, which a render reads for it; nil
|
||||||
|
// for a builtin that reads none.
|
||||||
|
operands func(args []string) []string
|
||||||
|
// proveNumber bounds the number a call's text reads as, token its body, and says which
|
||||||
|
// datatypes that text is not; set it where every render reads as a finite number, which
|
||||||
|
// makes the call a calc operand, and leave it nil otherwise.
|
||||||
|
proveNumber func(token string, prints datatype.DataType, args []string) proven.Facts
|
||||||
|
// prints is the datatype a call's text is, handed to proveNumber: datatype.String where it
|
||||||
|
// reads as a number no column should type, as digits' leading zeros; unset where
|
||||||
|
// proveNumber is nil.
|
||||||
|
prints datatype.DataType
|
||||||
|
}
|
||||||
|
|
||||||
|
// CalcName is the builtin whose operands package fejkdata checks and proves, since only it
|
||||||
|
// holds the nodes they read.
|
||||||
|
const CalcName = "calc"
|
||||||
|
|
||||||
|
// rng is the randomness a builtin sample draws from, which *drawstate.State satisfies.
|
||||||
|
type rng interface {
|
||||||
|
IntN(n int) int
|
||||||
|
Float64() float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check validates a call at compile time: naming a known builtin, with the arg count that
|
||||||
|
// builtin takes and args its check accepts.
|
||||||
|
func Check(name string, args []string) error {
|
||||||
|
b, known := builtins[name]
|
||||||
|
if !known {
|
||||||
|
return fmt.Errorf("unknown function %q", name)
|
||||||
|
}
|
||||||
|
if b.arity >= 0 && len(args) != b.arity {
|
||||||
|
return fmt.Errorf("%s takes %d argument%s, got %d", name, b.arity, plural(b.arity), len(args))
|
||||||
|
}
|
||||||
|
if b.checkArgs != nil {
|
||||||
|
return b.checkArgs(args)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prep parses a call's args, which Check accepted, into the Call a render makes.
|
||||||
|
func Prep(name string, args []string) Call { return builtins[name].prep(args) }
|
||||||
|
|
||||||
|
// Operands lists the fields a call reads as operands, empty for a builtin that reads none
|
||||||
|
// or is unknown.
|
||||||
|
func Operands(name string, args []string) []string {
|
||||||
|
b, known := builtins[name]
|
||||||
|
if !known || b.operands == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return b.operands(args)
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsTransform reports a builtin that rewrites the text of its one operand.
|
||||||
|
func IsTransform(name string) bool {
|
||||||
|
_, isTransform := transforms[name]
|
||||||
|
return isTransform
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProveNumber proves a call whose every render reads as a finite number, token its body;
|
||||||
|
// false for any other call, calc included: proven.OfCalc proves a calc from its operands,
|
||||||
|
// which only the caller holds.
|
||||||
|
func ProveNumber(name, token string, args []string) (proven.Facts, bool) {
|
||||||
|
b := builtins[name]
|
||||||
|
if b.proveNumber == nil {
|
||||||
|
return proven.Facts{}, false
|
||||||
|
}
|
||||||
|
return b.proveNumber(token, b.prints, args), true
|
||||||
|
}
|
||||||
|
|
||||||
|
// TypedCalls lists the calls whose text a typed column may hold, as an error names them.
|
||||||
|
func TypedCalls() string { return numberCalls(false) }
|
||||||
|
|
||||||
|
// OperandCalls lists the calls whose text a calc reads as a number, as an error names them.
|
||||||
|
func OperandCalls() string { return numberCalls(true) }
|
||||||
|
|
||||||
|
func numberCalls(text bool) string {
|
||||||
|
var calls []string
|
||||||
|
for name, b := range builtins {
|
||||||
|
if b.proveNumber != nil && (text || b.prints != datatype.String) {
|
||||||
|
calls = append(calls, "{"+name+"()}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
slices.Sort(calls)
|
||||||
|
return strings.Join(calls, ", ") + " or {" + CalcName + "()}"
|
||||||
|
}
|
||||||
|
|
||||||
|
func plural(n int) string {
|
||||||
|
if n == 1 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return "s"
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
package builtinfunc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata/internal/invariant"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestRegistryShapes pins the builtin contract Prep relies on: every entry supplies prep,
|
||||||
|
// and args parsed at compile only behind a check.
|
||||||
|
func TestRegistryShapes(t *testing.T) {
|
||||||
|
for name, b := range builtins {
|
||||||
|
if b.prep == nil {
|
||||||
|
t.Errorf("builtin %q has no prep: Prep would call a nil func", name)
|
||||||
|
}
|
||||||
|
if b.arity != 0 && b.checkArgs == nil {
|
||||||
|
t.Errorf("builtin %q parses args in prep with no check", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckNamesWhatACallGetsWrong(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"nope", nil, `unknown function "nope"`},
|
||||||
|
{"luhn", []string{"1"}, "luhn takes 0 arguments, got 1"},
|
||||||
|
{"hex", nil, "hex takes 1 argument, got 0"},
|
||||||
|
{"calc", []string{"a +"}, `calc("a +"): `},
|
||||||
|
{"calc", []string{"a", "-1"}, "calc decimals "},
|
||||||
|
{"calc", nil, "calc takes an expression and an optional decimals count, got 0 args"},
|
||||||
|
} {
|
||||||
|
if err := Check(c.name, c.args); err == nil || !strings.HasPrefix(err.Error(), c.want) {
|
||||||
|
t.Errorf("Check(%s, %q) = %v, want it to start %q", c.name, c.args, err, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := Check("calc", []string{"a / b", "2"}); err != nil {
|
||||||
|
t.Errorf("Check(calc) = %v, want an expression over any operand accepted", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestArgGuardsPanic pins the guards that report a builtin arg its check should have
|
||||||
|
// rejected. No data reaches them, since Check runs before Prep, so they are called directly.
|
||||||
|
func TestArgGuardsPanic(t *testing.T) {
|
||||||
|
for name, call := range map[string]func(){
|
||||||
|
"atoi on an unvalidated arg": func() { atoi("nope") },
|
||||||
|
"atof on an unvalidated arg": func() { atof("nope") },
|
||||||
|
"prep on an expression that won't parse": func() { calcPrep([]string{"1 +"}) },
|
||||||
|
} {
|
||||||
|
mustPanic(t, name, call)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// mustPanic fails unless call panics with invariant.Broken's phrase, which is what
|
||||||
|
// separates a reported invariant break from a silently wrong value.
|
||||||
|
func mustPanic(t *testing.T, name string, call func()) {
|
||||||
|
t.Helper()
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r == nil || !strings.HasPrefix(fmt.Sprint(r), invariant.Broken("")) {
|
||||||
|
t.Errorf("%s: recovered %v, want the invariant reported as an internal error", name, r)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
call()
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package fejkdata
|
package builtinfunc
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
@@ -7,6 +7,11 @@ import (
|
|||||||
"math"
|
"math"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata/internal/datatype"
|
||||||
|
"github.com/larvit/fejkdata/internal/drawstate"
|
||||||
|
"github.com/larvit/fejkdata/internal/invariant"
|
||||||
|
"github.com/larvit/fejkdata/internal/proven"
|
||||||
)
|
)
|
||||||
|
|
||||||
// maxLen caps sample output lengths (hex, nanoid, base64, digits, upper, lower)
|
// maxLen caps sample output lengths (hex, nanoid, base64, digits, upper, lower)
|
||||||
@@ -22,88 +27,88 @@ const (
|
|||||||
// emitted so far in the current expansion (place them after their payload);
|
// emitted so far in the current expansion (place them after their payload);
|
||||||
// samples read only the rng. A time-based id (uuid v7, ulid) draws its timestamp
|
// samples read only the rng. A time-based id (uuid v7, ulid) draws its timestamp
|
||||||
// from the rng, not the wall clock, so seeded output stays reproducible.
|
// from the rng, not the wall clock, so seeded output stays reproducible.
|
||||||
var builtins = map[string]builtin{
|
var builtins = withTransforms(map[string]builtin{
|
||||||
"luhn": {arity: 0, prep: derive(func(e string) string { return string(rune('0' + luhnCheck(e))) })},
|
"luhn": {arity: 0, prep: derive(func(e string) string { return string(rune('0' + luhnCheck(e))) })},
|
||||||
"mod11": {arity: 0, prep: derive(mod11Check)},
|
"mod11": {arity: 0, prep: derive(mod11Check)},
|
||||||
"ean": {arity: 0, prep: derive(eanCheck)},
|
"ean": {arity: 0, prep: derive(eanCheck)},
|
||||||
"uuid": {arity: 0, prep: sample(uuidV7)},
|
"uuid": {arity: 0, prep: sample(uuidV7)},
|
||||||
"ulid": {arity: 0, prep: sample(ulid)},
|
"ulid": {arity: 0, prep: sample(ulid)},
|
||||||
"nanoid": {arity: 1, check: posIntArg, prep: chars(nanoidAlphabet)},
|
"nanoid": {arity: 1, checkArgs: countArg, prep: chars(nanoidAlphabet)},
|
||||||
"hex": {arity: 1, check: posIntArg, prep: chars(hexDigits)},
|
"hex": {arity: 1, checkArgs: countArg, prep: chars(hexDigits)},
|
||||||
"digits": {arity: 1, check: posIntArg, prep: chars("0123456789"), number: func(token string, a []string) proven {
|
"digits": {arity: 1, checkArgs: countArg, prep: chars("0123456789"), prints: datatype.String, proveNumber: func(token string, prints datatype.DataType, a []string) proven.Facts {
|
||||||
return printing(token, DataTypeString, bounded(0, math.Pow(10, float64(atoi(a[0])))-1, true))
|
if atoi(a[0]) == 0 {
|
||||||
|
return proven.Unproven("{" + token + "} prints nothing, which is no number")
|
||||||
|
}
|
||||||
|
return proven.Printing(token, prints, proven.Bounded(0, math.Pow(10, float64(atoi(a[0])))-1, true))
|
||||||
}},
|
}},
|
||||||
"upper": {arity: 1, check: posIntArg, prep: chars("ABCDEFGHIJKLMNOPQRSTUVWXYZ")},
|
"upper": {arity: 1, checkArgs: countArg, prep: chars("ABCDEFGHIJKLMNOPQRSTUVWXYZ")},
|
||||||
"lower": {arity: 1, check: posIntArg, prep: chars("abcdefghijklmnopqrstuvwxyz")},
|
"lower": {arity: 1, checkArgs: countArg, prep: chars("abcdefghijklmnopqrstuvwxyz")},
|
||||||
"base64": {arity: 1, check: posIntArg, prep: func(a []string) callFn {
|
"base64": {arity: 1, checkArgs: countArg, prep: func(a []string) Call {
|
||||||
n := atoi(a[0])
|
n := atoi(a[0])
|
||||||
return func(s *session, _ string, _ []string) string {
|
return func(s *drawstate.State, _ string, _ []string) string {
|
||||||
return base64.StdEncoding.EncodeToString(randBytes(s, n))
|
return base64.StdEncoding.EncodeToString(randBytes(s, n))
|
||||||
}
|
}
|
||||||
}},
|
}},
|
||||||
"int": {arity: 2, check: intRangeArgs, prep: func(a []string) callFn {
|
"int": {arity: 2, checkArgs: intRangeArgs, prep: func(a []string) Call {
|
||||||
lo, span := atoi(a[0]), atoi(a[1])-atoi(a[0])+1
|
lo, span := atoi(a[0]), atoi(a[1])-atoi(a[0])+1
|
||||||
return func(s *session, _ string, _ []string) string { return strconv.Itoa(lo + s.IntN(span)) }
|
return func(s *drawstate.State, _ string, _ []string) string { return strconv.Itoa(lo + s.IntN(span)) }
|
||||||
}, number: func(token string, a []string) proven {
|
}, prints: datatype.Integer, proveNumber: func(token string, prints datatype.DataType, a []string) proven.Facts {
|
||||||
return printing(token, DataTypeInteger, bounded(float64(atoi(a[0])), float64(atoi(a[1])), true))
|
return proven.Printing(token, prints, proven.Bounded(float64(atoi(a[0])), float64(atoi(a[1])), true))
|
||||||
}},
|
}},
|
||||||
"float": {arity: 3, check: floatArgs, prep: func(a []string) callFn {
|
"float": {arity: 3, checkArgs: floatArgs, prep: func(a []string) Call {
|
||||||
lo, hi, dp := atof(a[0]), atof(a[1]), atoi(a[2])
|
lo, hi, dp := atof(a[0]), atof(a[1]), atoi(a[2])
|
||||||
return func(s *session, _ string, _ []string) string {
|
return func(s *drawstate.State, _ string, _ []string) string {
|
||||||
return formatFloat(lo+s.Float64()*(hi-lo), dp)
|
return formatFloat(lo+s.Float64()*(hi-lo), dp)
|
||||||
}
|
}
|
||||||
}, number: func(token string, a []string) proven {
|
}, prints: datatype.Number, proveNumber: func(token string, _ datatype.DataType, a []string) proven.Facts {
|
||||||
return printedNumber(token, bounded(atof(a[0]), atof(a[1]), false), atoi(a[2]))
|
return proven.PrintedNumber(token, proven.Bounded(atof(a[0]), atof(a[1]), false), atoi(a[2]))
|
||||||
}},
|
}},
|
||||||
"iban": {arity: 1, check: ibanArg, prep: func(a []string) callFn {
|
"iban": {arity: 1, checkArgs: ibanArg, prep: func(a []string) Call {
|
||||||
cc := a[0]
|
cc := a[0]
|
||||||
return func(s *session, _ string, _ []string) string { return iban(s, cc) }
|
return func(s *drawstate.State, _ string, _ []string) string { return iban(s, cc) }
|
||||||
}},
|
}},
|
||||||
"date": {arity: -1, check: dateArgs, prep: datePrep},
|
"date": {arity: -1, checkArgs: dateArgs, prep: datePrep},
|
||||||
"time": {arity: -1, check: timeArg, prep: timePrep},
|
"time": {arity: -1, checkArgs: timeArg, prep: timePrep},
|
||||||
"calc": {arity: -1, check: checkCalc, prep: calcPrep, operands: calcOperands},
|
CalcName: {arity: -1, checkArgs: checkCalc, prep: calcPrep, operands: calcOperands},
|
||||||
"lowercase": {arity: 1, check: transformArg, prep: transformPrep(strings.ToLower), operands: transformOperand},
|
// seq is the one stateful builtin: a per-generator counter from 1, advancing on
|
||||||
"uppercase": {arity: 1, check: transformArg, prep: transformPrep(strings.ToUpper), operands: transformOperand},
|
|
||||||
"ascii": {arity: 1, check: transformArg, prep: transformPrep(asciiFold), operands: transformOperand},
|
|
||||||
// seq is the one stateful builtin: a per-session counter from 1, advancing on
|
|
||||||
// each call. An optional name selects an independent counter; no name uses the
|
// each call. An optional name selects an independent counter; no name uses the
|
||||||
// default one. Deterministic by construction, so seeded output stays stable.
|
// default one. Deterministic by construction, so seeded output stays stable.
|
||||||
"seq": {arity: -1, check: seqArg, prep: func(a []string) callFn {
|
"seq": {arity: -1, checkArgs: seqArg, prep: func(a []string) Call {
|
||||||
key := ""
|
key := ""
|
||||||
if len(a) == 1 {
|
if len(a) == 1 {
|
||||||
key = a[0]
|
key = a[0]
|
||||||
}
|
}
|
||||||
return func(s *session, _ string, _ []string) string {
|
return func(s *drawstate.State, _ string, _ []string) string {
|
||||||
return strconv.FormatUint(s.next(key), 10)
|
return strconv.FormatUint(s.Seq(key), 10)
|
||||||
}
|
}
|
||||||
}, number: func(token string, _ []string) proven {
|
}, prints: datatype.Integer, proveNumber: func(token string, prints datatype.DataType, _ []string) proven.Facts {
|
||||||
return printing(token, DataTypeInteger, bounded(1, math.MaxInt64, true))
|
return proven.Printing(token, prints, proven.Bounded(1, math.MaxInt64, true))
|
||||||
}},
|
}},
|
||||||
}
|
})
|
||||||
|
|
||||||
// derive and sample are the two argument-free builtin shapes: a derivation reads
|
// derive and sample are the two argument-free builtin shapes: a derivation reads
|
||||||
// the output emitted so far, a sample reads only the rng. chars is the shape of a
|
// the output emitted so far, a sample reads only the rng. chars is the shape of a
|
||||||
// sample of n characters drawn from an alphabet.
|
// sample of n characters drawn from an alphabet.
|
||||||
func derive(f func(emitted string) string) func([]string) callFn {
|
func derive(f func(emitted string) string) func([]string) Call {
|
||||||
return func([]string) callFn {
|
return func([]string) Call {
|
||||||
return func(_ *session, emitted string, _ []string) string { return f(emitted) }
|
return func(_ *drawstate.State, emitted string, _ []string) string { return f(emitted) }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
func sample(f func(rng) string) func([]string) callFn {
|
func sample(f func(rng) string) func([]string) Call {
|
||||||
return func([]string) callFn {
|
return func([]string) Call {
|
||||||
return func(s *session, _ string, _ []string) string { return f(s) }
|
return func(s *drawstate.State, _ string, _ []string) string { return f(s) }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
func chars(alphabet string) func([]string) callFn {
|
func chars(alphabet string) func([]string) Call {
|
||||||
return func(a []string) callFn {
|
return func(a []string) Call {
|
||||||
n := atoi(a[0])
|
n := atoi(a[0])
|
||||||
return func(s *session, _ string, _ []string) string { return randChars(s, n, alphabet) }
|
return func(s *drawstate.State, _ string, _ []string) string { return randChars(s, n, alphabet) }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const hexDigits = "0123456789abcdef"
|
const hexDigits = "0123456789abcdef"
|
||||||
|
|
||||||
// formatFloat prints v to dp decimals, -1 for the shortest form, and a zero unsigned.
|
// formatFloat prints v to dp decimals, or proven.ShortestDecimals, and a zero unsigned.
|
||||||
func formatFloat(v float64, dp int) string {
|
func formatFloat(v float64, dp int) string {
|
||||||
s := strconv.FormatFloat(v, 'f', dp, 64)
|
s := strconv.FormatFloat(v, 'f', dp, 64)
|
||||||
if strings.HasPrefix(s, "-") && strings.Trim(s, "-0.") == "" {
|
if strings.HasPrefix(s, "-") && strings.Trim(s, "-0.") == "" {
|
||||||
@@ -118,7 +123,7 @@ func formatFloat(v float64, dp int) string {
|
|||||||
func atoi(s string) int {
|
func atoi(s string) int {
|
||||||
n, err := strconv.Atoi(s)
|
n, err := strconv.Atoi(s)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", s, err))
|
panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", s, err))
|
||||||
}
|
}
|
||||||
return n
|
return n
|
||||||
}
|
}
|
||||||
@@ -127,7 +132,7 @@ func atoi(s string) int {
|
|||||||
func atof(s string) float64 {
|
func atof(s string) float64 {
|
||||||
f, err := strconv.ParseFloat(s, 64)
|
f, err := strconv.ParseFloat(s, 64)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", s, err))
|
panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", s, err))
|
||||||
}
|
}
|
||||||
return f
|
return f
|
||||||
}
|
}
|
||||||
@@ -146,8 +151,7 @@ func randChars(r rng, n int, alphabet string) string {
|
|||||||
return string(b)
|
return string(b)
|
||||||
}
|
}
|
||||||
|
|
||||||
// plainInt parses an integer arg written the one way: no sign, no leading zero.
|
func intArg(s string) (int, error) {
|
||||||
func plainInt(s string) (int, error) {
|
|
||||||
n, err := strconv.Atoi(s)
|
n, err := strconv.Atoi(s)
|
||||||
if errors.Is(err, strconv.ErrRange) {
|
if errors.Is(err, strconv.ErrRange) {
|
||||||
return 0, fmt.Errorf("%q is past the integer range: %w", s, err)
|
return 0, fmt.Errorf("%q is past the integer range: %w", s, err)
|
||||||
@@ -155,54 +159,48 @@ func plainInt(s string) (int, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, fmt.Errorf("%q is not an integer", s)
|
return 0, fmt.Errorf("%q is not an integer", s)
|
||||||
}
|
}
|
||||||
if strconv.Itoa(n) != s {
|
|
||||||
return 0, fmt.Errorf("%q is not a plain integer; write %d", s, n)
|
|
||||||
}
|
|
||||||
return n, nil
|
return n, nil
|
||||||
}
|
}
|
||||||
func posIntArg(_ map[string]node, a []string) error {
|
func countArg(a []string) error {
|
||||||
n, err := plainInt(a[0])
|
n, err := intArg(a[0])
|
||||||
if errors.Is(err, strconv.ErrRange) {
|
if errors.Is(err, strconv.ErrRange) {
|
||||||
return fmt.Errorf("count %q exceeds the maximum %d", a[0], maxLen)
|
return fmt.Errorf("count %q exceeds the maximum %d", a[0], maxLen)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("count %w", err)
|
return fmt.Errorf("count %w", err)
|
||||||
}
|
}
|
||||||
if n < 1 {
|
if n < 0 {
|
||||||
return fmt.Errorf("count %q must be positive", a[0])
|
return fmt.Errorf("count %q must not be negative", a[0])
|
||||||
}
|
}
|
||||||
if n > maxLen {
|
if n > maxLen {
|
||||||
return fmt.Errorf("count %d exceeds the maximum %d", n, maxLen)
|
return fmt.Errorf("count %d exceeds the maximum %d", n, maxLen)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
func intRangeArgs(_ map[string]node, a []string) error {
|
func intRangeArgs(a []string) error {
|
||||||
lo, err := plainInt(a[0])
|
lo, err := intArg(a[0])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("int(min,max): min %w", err)
|
return fmt.Errorf("int(min,max): min %w", err)
|
||||||
}
|
}
|
||||||
hi, err := plainInt(a[1])
|
hi, err := intArg(a[1])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("int(min,max): max %w", err)
|
return fmt.Errorf("int(min,max): max %w", err)
|
||||||
}
|
}
|
||||||
if lo > hi {
|
if lo > hi {
|
||||||
return fmt.Errorf("int(min,max): min %d > max %d", lo, hi)
|
return fmt.Errorf("int(min,max): min %d > max %d", lo, hi)
|
||||||
}
|
}
|
||||||
if lo == hi {
|
|
||||||
return fmt.Errorf("int(%d,%d) is the constant %d; write it as text", lo, hi, lo)
|
|
||||||
}
|
|
||||||
if uint64(hi)-uint64(lo) >= uint64(math.MaxInt64) { // span hi-lo+1 would overflow int -> IntN panic
|
if uint64(hi)-uint64(lo) >= uint64(math.MaxInt64) { // span hi-lo+1 would overflow int -> IntN panic
|
||||||
return fmt.Errorf("int(min,max): range %d..%d is too wide", lo, hi)
|
return fmt.Errorf("int(min,max): range %d..%d is too wide", lo, hi)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
func floatArgs(_ map[string]node, a []string) error {
|
func floatArgs(a []string) error {
|
||||||
lo, e1 := strconv.ParseFloat(a[0], 64)
|
lo, e1 := strconv.ParseFloat(a[0], 64)
|
||||||
hi, e2 := strconv.ParseFloat(a[1], 64)
|
hi, e2 := strconv.ParseFloat(a[1], 64)
|
||||||
if e1 != nil || e2 != nil {
|
if e1 != nil || e2 != nil {
|
||||||
return fmt.Errorf("float(min,max,dp) needs numeric bounds, got %q,%q", a[0], a[1])
|
return fmt.Errorf("float(min,max,dp) needs numeric bounds, got %q,%q", a[0], a[1])
|
||||||
}
|
}
|
||||||
dp, err := plainInt(a[2])
|
dp, err := intArg(a[2])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("float(min,max,dp): decimals %w", err)
|
return fmt.Errorf("float(min,max,dp): decimals %w", err)
|
||||||
}
|
}
|
||||||
@@ -215,15 +213,12 @@ func floatArgs(_ map[string]node, a []string) error {
|
|||||||
if dp < 0 || dp > maxDecimals {
|
if dp < 0 || dp > maxDecimals {
|
||||||
return fmt.Errorf("float(min,max,dp): decimals %d out of range 0..%d", dp, maxDecimals)
|
return fmt.Errorf("float(min,max,dp): decimals %d out of range 0..%d", dp, maxDecimals)
|
||||||
}
|
}
|
||||||
if lo == hi {
|
|
||||||
return fmt.Errorf("float(%s,%s,%d) is the constant %q; write it as text", a[0], a[1], dp, strconv.FormatFloat(lo, 'f', dp, 64))
|
|
||||||
}
|
|
||||||
if math.IsInf(hi-lo, 0) { // an overflowing span would render as "+Inf"
|
if math.IsInf(hi-lo, 0) { // an overflowing span would render as "+Inf"
|
||||||
return fmt.Errorf("float(min,max,dp): range %v..%v is too wide", lo, hi)
|
return fmt.Errorf("float(min,max,dp): range %v..%v is too wide", lo, hi)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
func seqArg(_ map[string]node, a []string) error {
|
func seqArg(a []string) error {
|
||||||
if len(a) > 1 {
|
if len(a) > 1 {
|
||||||
return fmt.Errorf("seq takes at most one name, got %d", len(a))
|
return fmt.Errorf("seq takes at most one name, got %d", len(a))
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
package builtinfunc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata/internal/drawstate"
|
||||||
|
"github.com/larvit/fejkdata/internal/grammar"
|
||||||
|
"github.com/larvit/fejkdata/internal/invariant"
|
||||||
|
"github.com/larvit/fejkdata/internal/proven"
|
||||||
|
)
|
||||||
|
|
||||||
|
// evalCalc evaluates an expression over the operand values a render already read, so it
|
||||||
|
// draws nothing.
|
||||||
|
func evalCalc(n grammar.CalcNode, operands []string) float64 {
|
||||||
|
switch n := n.(type) {
|
||||||
|
case grammar.CalcNum:
|
||||||
|
return float64(n)
|
||||||
|
case grammar.CalcVar:
|
||||||
|
v, err := strconv.ParseFloat(strings.TrimSpace(operands[n.At]), 64)
|
||||||
|
if err != nil {
|
||||||
|
return math.NaN() // a non-numeric operand stays visible, never an error
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
case grammar.CalcNeg:
|
||||||
|
return -evalCalc(n.X, operands)
|
||||||
|
case grammar.CalcBin:
|
||||||
|
return Arith(n.Operator, evalCalc(n.L, operands), evalCalc(n.R, operands))
|
||||||
|
}
|
||||||
|
panic(invariant.Broken("calc node %#v has no evaluation", n))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Arith applies a calc operator.
|
||||||
|
func Arith(operator byte, l, r float64) float64 {
|
||||||
|
switch operator {
|
||||||
|
case '+':
|
||||||
|
return l + r
|
||||||
|
case '-':
|
||||||
|
return l - r
|
||||||
|
case '*':
|
||||||
|
return l * r
|
||||||
|
default: // '/'
|
||||||
|
return l / r
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkCalc validates a calc token's own args: a parseable expression and an optional
|
||||||
|
// non-negative integer dp. What its operands hold is the caller's to check.
|
||||||
|
func checkCalc(args []string) error {
|
||||||
|
if len(args) < 1 || len(args) > 2 {
|
||||||
|
return fmt.Errorf("calc takes an expression and an optional decimals count, got %d args", len(args))
|
||||||
|
}
|
||||||
|
if _, err := grammar.ParseCalc(args[0]); err != nil {
|
||||||
|
return fmt.Errorf("calc(%q): %w", args[0], err)
|
||||||
|
}
|
||||||
|
if len(args) == 2 {
|
||||||
|
dp, err := intArg(args[1])
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("calc decimals %w", err)
|
||||||
|
}
|
||||||
|
if dp < 0 || dp > maxDecimals {
|
||||||
|
return fmt.Errorf("calc decimals %d must be in 0..%d", dp, maxDecimals)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// calcPrep parses the expression and decimals once, at compile time. checkCalc proved
|
||||||
|
// both args valid, so no step here can fail.
|
||||||
|
func calcPrep(args []string) Call {
|
||||||
|
expr := ParsedCalc(args[0]).Expr
|
||||||
|
dp := CalcDecimals(args)
|
||||||
|
return func(_ *drawstate.State, _ string, operands []string) string {
|
||||||
|
return formatFloat(evalCalc(expr, operands), dp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParsedCalc parses an expression Check accepted.
|
||||||
|
func ParsedCalc(expr string) grammar.Calc {
|
||||||
|
c, err := grammar.ParseCalc(expr)
|
||||||
|
if err != nil {
|
||||||
|
panic(invariant.Broken("calc(%q) passed its check unparsed: %v", expr, err))
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// CalcDecimals is the decimals count of a calc Check accepted, or proven.ShortestDecimals
|
||||||
|
// where it names none.
|
||||||
|
func CalcDecimals(args []string) int {
|
||||||
|
if len(args) == 2 {
|
||||||
|
return atoi(args[1])
|
||||||
|
}
|
||||||
|
return proven.ShortestDecimals
|
||||||
|
}
|
||||||
|
|
||||||
|
// calcOperands lists the operands a calc's args read, fields or names. checkCalc reports
|
||||||
|
// an expression that does not parse, so one that does not simply names nothing.
|
||||||
|
func calcOperands(args []string) []string {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
c, err := grammar.ParseCalc(args[0])
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return c.Operands
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package fejkdata
|
package builtinfunc
|
||||||
|
|
||||||
import "fmt"
|
import "fmt"
|
||||||
|
|
||||||
@@ -65,16 +65,14 @@ func eanCheck(s string) string {
|
|||||||
// a left-to-right derivation; it generates the whole value instead.
|
// a left-to-right derivation; it generates the whole value instead.
|
||||||
var ibanLen = map[string]int{"BE": 16, "DE": 22, "DK": 18, "ES": 24, "FI": 18, "NO": 15, "SE": 24}
|
var ibanLen = map[string]int{"BE": 16, "DE": 22, "DK": 18, "ES": 24, "FI": 18, "NO": 15, "SE": 24}
|
||||||
|
|
||||||
func ibanArg(_ map[string]node, a []string) error {
|
func ibanArg(a []string) error {
|
||||||
if _, ok := ibanLen[a[0]]; !ok {
|
if _, ok := ibanLen[a[0]]; !ok {
|
||||||
return fmt.Errorf("iban(%q): unsupported country code", a[0])
|
return fmt.Errorf("iban(%q): unsupported country code", a[0])
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// iban generates a structurally valid IBAN for cc: a numeric BBAN of the right
|
// iban generates an IBAN for cc: a numeric BBAN of the right length, then mod-97 check digits.
|
||||||
// length, then mod-97 check digits. Real bank/branch structure isn't modelled —
|
|
||||||
// the result passes length and checksum validation, which is what fake data needs.
|
|
||||||
func iban(r rng, cc string) string {
|
func iban(r rng, cc string) string {
|
||||||
bban := make([]byte, ibanLen[cc]-4)
|
bban := make([]byte, ibanLen[cc]-4)
|
||||||
for i := range bban {
|
for i := range bban {
|
||||||
@@ -1,39 +1,23 @@
|
|||||||
package fejkdata
|
package builtinfunc
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata/internal/drawstate"
|
||||||
|
"github.com/larvit/fejkdata/internal/invariant"
|
||||||
)
|
)
|
||||||
|
|
||||||
const dayLayout = "2006-01-02"
|
const dayLayout = "2006-01-02"
|
||||||
|
|
||||||
// The instants a layout is proved against: layoutProbe2 is alike in no field, while
|
|
||||||
// layoutDay differs from layoutProbe in its date fields alone and layoutClock in its
|
|
||||||
// clock fields alone, so formatting two of them tells which kind a layout names.
|
|
||||||
var (
|
|
||||||
layoutProbe = time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
|
|
||||||
layoutProbe2 = time.Date(2010, 11, 12, 13, 14, 15, 0, time.UTC)
|
|
||||||
layoutDay = time.Date(2010, 11, 12, 4, 5, 6, 0, time.UTC)
|
|
||||||
layoutClock = time.Date(2001, 2, 3, 13, 14, 15, 0, time.UTC)
|
|
||||||
)
|
|
||||||
|
|
||||||
func namesAField(layout string) bool {
|
|
||||||
return layoutProbe.Format(layout) != layoutProbe2.Format(layout)
|
|
||||||
}
|
|
||||||
func namesADateField(layout string) bool {
|
|
||||||
return layoutProbe.Format(layout) != layoutDay.Format(layout)
|
|
||||||
}
|
|
||||||
func namesAClockField(layout string) bool {
|
|
||||||
return layoutProbe.Format(layout) != layoutClock.Format(layout)
|
|
||||||
}
|
|
||||||
|
|
||||||
// quotedLayout reports whether an arg carries the single quotes a layout is written in.
|
// quotedLayout reports whether an arg carries the single quotes a layout is written in.
|
||||||
func quotedLayout(a string) bool {
|
func quotedLayout(a string) bool {
|
||||||
return len(a) >= 2 && a[0] == '\'' && a[len(a)-1] == '\''
|
return len(a) >= 2 && a[0] == '\'' && a[len(a)-1] == '\''
|
||||||
}
|
}
|
||||||
|
|
||||||
// layoutArg is the Go layout a quoted arg holds, refused when unquoted or constant.
|
// layoutArg is the Go layout a quoted arg holds, refused when unquoted.
|
||||||
|
// docs/decisions.md#a-layout-is-always-quoted
|
||||||
func layoutArg(a string) (string, error) {
|
func layoutArg(a string) (string, error) {
|
||||||
if !quotedLayout(a) {
|
if !quotedLayout(a) {
|
||||||
bare := strings.Trim(a, `'"`)
|
bare := strings.Trim(a, `'"`)
|
||||||
@@ -42,18 +26,14 @@ func layoutArg(a string) (string, error) {
|
|||||||
}
|
}
|
||||||
return "", fmt.Errorf("layout %s is not quoted; write '%s'", a, bare)
|
return "", fmt.Errorf("layout %s is not quoted; write '%s'", a, bare)
|
||||||
}
|
}
|
||||||
layout := a[1 : len(a)-1]
|
return a[1 : len(a)-1], nil
|
||||||
if !namesAField(layout) {
|
|
||||||
return "", fmt.Errorf("layout '%s' names no field, so it is the constant %q; write it as text", layout, layout)
|
|
||||||
}
|
|
||||||
return layout, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// layoutOf is layoutArg for an arg a check already validated.
|
// layoutOf is layoutArg for an arg a check already validated.
|
||||||
func layoutOf(a string) string {
|
func layoutOf(a string) string {
|
||||||
layout, err := layoutArg(a)
|
layout, err := layoutArg(a)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", a, err))
|
panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", a, err))
|
||||||
}
|
}
|
||||||
return layout
|
return layout
|
||||||
}
|
}
|
||||||
@@ -81,7 +61,7 @@ func holdsQuotedLayout(a []string) bool {
|
|||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
func dateArgs(_ map[string]node, a []string) error {
|
func dateArgs(a []string) error {
|
||||||
if err := layoutArity("date", 3, a); err != nil {
|
if err := layoutArity("date", 3, a); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -96,51 +76,45 @@ func dateArgs(_ map[string]node, a []string) error {
|
|||||||
if to.Before(from) {
|
if to.Before(from) {
|
||||||
return fmt.Errorf("date(from,to,layout): from %s is after to %s", a[0], a[1])
|
return fmt.Errorf("date(from,to,layout): from %s is after to %s", a[0], a[1])
|
||||||
}
|
}
|
||||||
layout, err := layoutArg(a[2])
|
_, err = layoutArg(a[2])
|
||||||
if err != nil {
|
return err
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !namesADateField(layout) {
|
|
||||||
return fmt.Errorf("date(from,to,layout): '%s' names no date field; write time('%s')", layout, layout)
|
|
||||||
}
|
|
||||||
if from.Equal(to) && !namesAClockField(layout) {
|
|
||||||
return fmt.Errorf("date(%s,%s,'%s') is the constant %q; write it as text", a[0], a[1], layout, from.Format(layout))
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
func timeArg(_ map[string]node, a []string) error {
|
func timeArg(a []string) error {
|
||||||
if err := layoutArity("time", 1, a); err != nil {
|
if err := layoutArity("time", 1, a); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
layout, err := layoutArg(a[0])
|
layout, err := layoutArg(a[0])
|
||||||
if err != nil {
|
if err == nil && namesADateField(layout) {
|
||||||
return err
|
return fmt.Errorf("time(layout): '%s' names a date field, and time draws no date; write date(from,to,layout)", layout)
|
||||||
}
|
}
|
||||||
if namesADateField(layout) {
|
return err
|
||||||
return fmt.Errorf("time(layout): '%s' names a date field; write date(from,to,layout)", layout)
|
}
|
||||||
}
|
|
||||||
return nil
|
// namesADateField reports a layout that prints two instants differing in their date alone apart.
|
||||||
|
func namesADateField(layout string) bool {
|
||||||
|
day, nextYear := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC), time.Date(2010, 11, 12, 4, 5, 6, 0, time.UTC)
|
||||||
|
return day.Format(layout) != nextYear.Format(layout)
|
||||||
}
|
}
|
||||||
|
|
||||||
// datePrep draws a second in [from 00:00:00, to 23:59:59] UTC; the span is counted
|
// datePrep draws a second in [from 00:00:00, to 23:59:59] UTC; the span is counted
|
||||||
// in seconds, since a Duration overflows past 292 years.
|
// in seconds, since a Duration overflows past 292 years.
|
||||||
func datePrep(a []string) callFn {
|
func datePrep(a []string) Call {
|
||||||
from, err := time.Parse(dayLayout, a[0])
|
from, err := time.Parse(dayLayout, a[0])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", a[0], err))
|
panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", a[0], err))
|
||||||
}
|
}
|
||||||
to, err := time.Parse(dayLayout, a[1])
|
to, err := time.Parse(dayLayout, a[1])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(fmt.Sprintf("fejkdata: builtin arg %q reached prep unvalidated: %v", a[1], err))
|
panic(invariant.Broken("builtin arg %q reached prep unvalidated: %v", a[1], err))
|
||||||
}
|
}
|
||||||
layout, start, span := layoutOf(a[2]), from.Unix(), int(to.Unix()-from.Unix())+86400
|
layout, start, span := layoutOf(a[2]), from.Unix(), int(to.Unix()-from.Unix())+86400
|
||||||
return func(s *session, _ string, _ []string) string {
|
return func(s *drawstate.State, _ string, _ []string) string {
|
||||||
return time.Unix(start+int64(s.IntN(span)), 0).UTC().Format(layout)
|
return time.Unix(start+int64(s.IntN(span)), 0).UTC().Format(layout)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
func timePrep(a []string) callFn {
|
func timePrep(a []string) Call {
|
||||||
layout := layoutOf(a[0])
|
layout := layoutOf(a[0])
|
||||||
return func(s *session, _ string, _ []string) string {
|
return func(s *drawstate.State, _ string, _ []string) string {
|
||||||
return time.Unix(int64(s.IntN(86400)), 0).UTC().Format(layout)
|
return time.Unix(int64(s.IntN(86400)), 0).UTC().Format(layout)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,9 +1,13 @@
|
|||||||
package fejkdata
|
package builtinfunc
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"unicode"
|
"unicode"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata/internal/drawstate"
|
||||||
|
"github.com/larvit/fejkdata/internal/grammar"
|
||||||
|
"github.com/larvit/fejkdata/internal/invariant"
|
||||||
)
|
)
|
||||||
|
|
||||||
// transforms are the builtins that rewrite one operand's value; they nest, so
|
// transforms are the builtins that rewrite one operand's value; they nest, so
|
||||||
@@ -14,11 +18,21 @@ var transforms = map[string]func(string) string{
|
|||||||
"uppercase": strings.ToUpper,
|
"uppercase": strings.ToUpper,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func withTransforms(calls map[string]builtin) map[string]builtin {
|
||||||
|
for name, fn := range transforms {
|
||||||
|
if _, clash := calls[name]; clash {
|
||||||
|
panic(invariant.Broken("%s is registered as a builtin and a transform", name))
|
||||||
|
}
|
||||||
|
calls[name] = builtin{arity: 1, checkArgs: transformArg, prep: transformPrep(fn), operands: transformOperand}
|
||||||
|
}
|
||||||
|
return calls
|
||||||
|
}
|
||||||
|
|
||||||
// unwrapTransform peels nested transform calls off an operand arg, returning the
|
// unwrapTransform peels nested transform calls off an operand arg, returning the
|
||||||
// field it finally names and the transforms to apply, innermost last.
|
// field it finally names and the transforms to apply, innermost last.
|
||||||
func unwrapTransform(arg string) (leaf string, chain []func(string) string, err error) {
|
func unwrapTransform(arg string) (leaf string, chain []func(string) string, err error) {
|
||||||
for {
|
for {
|
||||||
name, args, isCall := funcCall(arg)
|
name, args, isCall := grammar.FuncCall(arg)
|
||||||
if !isCall {
|
if !isCall {
|
||||||
return arg, chain, nil
|
return arg, chain, nil
|
||||||
}
|
}
|
||||||
@@ -33,31 +47,29 @@ func unwrapTransform(arg string) (leaf string, chain []func(string) string, err
|
|||||||
arg = args[0]
|
arg = args[0]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
func transformArg(fields map[string]node, a []string) error {
|
|
||||||
leaf, _, err := unwrapTransform(a[0])
|
// transformArg checks that each call nested in the arg is a transform taking one arg.
|
||||||
if err != nil {
|
func transformArg(a []string) error {
|
||||||
return err
|
_, _, err := unwrapTransform(a[0])
|
||||||
}
|
return err
|
||||||
if isRef(leaf) {
|
|
||||||
_, _, err := refShape(leaf)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return checkArm(leaf, fields, false)
|
|
||||||
}
|
}
|
||||||
func transformOperand(a []string) []string {
|
func transformOperand(a []string) []string {
|
||||||
|
if len(a) != 1 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
leaf, _, err := unwrapTransform(a[0])
|
leaf, _, err := unwrapTransform(a[0])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return []string{leaf}
|
return []string{leaf}
|
||||||
}
|
}
|
||||||
func transformPrep(outer func(string) string) func([]string) callFn {
|
func transformPrep(outer func(string) string) func([]string) Call {
|
||||||
return func(a []string) callFn {
|
return func(a []string) Call {
|
||||||
_, chain, err := unwrapTransform(a[0])
|
_, chain, err := unwrapTransform(a[0])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(fmt.Sprintf("fejkdata: transform arg %q reached prep unvalidated: %v", a[0], err))
|
panic(invariant.Broken("transform arg %q reached prep unvalidated: %v", a[0], err))
|
||||||
}
|
}
|
||||||
return func(_ *session, _ string, operands []string) string {
|
return func(_ *drawstate.State, _ string, operands []string) string {
|
||||||
v := operands[0]
|
v := operands[0]
|
||||||
for i := len(chain) - 1; i >= 0; i-- {
|
for i := len(chain) - 1; i >= 0; i-- {
|
||||||
v = chain[i](v)
|
v = chain[i](v)
|
||||||
@@ -0,0 +1,215 @@
|
|||||||
|
// Package datafiles walks a data tree and reads its category files and their rows.
|
||||||
|
package datafiles
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata/internal/grammar"
|
||||||
|
"github.com/larvit/fejkdata/internal/jsonvalue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Source is one tree to load, from the root of an fs.FS. label prefixes file names in
|
||||||
|
// errors; onDisk marks label as a directory that must exist; at is the folder fsys mounts the
|
||||||
|
// tree at, "" for its root.
|
||||||
|
type Source struct {
|
||||||
|
fsys fs.FS
|
||||||
|
label string
|
||||||
|
onDisk bool
|
||||||
|
at string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dir is the tree in the directory path, its errors labelled with it.
|
||||||
|
func Dir(path string) Source {
|
||||||
|
return Source{fsys: os.DirFS(path), label: path, onDisk: true}
|
||||||
|
}
|
||||||
|
|
||||||
|
// FS is the tree in fsys.
|
||||||
|
func FS(fsys fs.FS) Source { return Source{fsys: fsys} }
|
||||||
|
|
||||||
|
// Mounted is s with its tree under the folders the dot path namespace spells, "" for none.
|
||||||
|
func (s Source) Mounted(namespace string) Source {
|
||||||
|
if namespace != "" && s.at == "" {
|
||||||
|
s.at = strings.ReplaceAll(namespace, ".", "/")
|
||||||
|
s.fsys = mounted{fsys: s.fsys, at: s.at}
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// Category is one category file: the folders above it from the source's root, its name,
|
||||||
|
// and its parsed JSON.
|
||||||
|
type Category struct {
|
||||||
|
Folders []string
|
||||||
|
Name string
|
||||||
|
JSON jsonvalue.Value
|
||||||
|
rows *rowsFiles
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadRows reads a rows file beside the category.
|
||||||
|
func (c Category) ReadRows(name string) (string, error) { return c.rows.read(name) }
|
||||||
|
|
||||||
|
// labelled is the file p as errors name it: a labelled source's as its own tree spells it,
|
||||||
|
// below the mount.
|
||||||
|
func (s Source) labelled(p string) string {
|
||||||
|
if s.label == "" {
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
if inner, below := underMount(s.at, p); below {
|
||||||
|
p = inner
|
||||||
|
}
|
||||||
|
return path.Join(s.label, p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// dirPath is the folder that dir names, as fsys spells it.
|
||||||
|
func (s Source) dirPath(dir []string) string {
|
||||||
|
if p := path.Join(dir...); p != "" {
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
return "."
|
||||||
|
}
|
||||||
|
|
||||||
|
// Walk hands compile every category of the tree, folders and files in name order. A
|
||||||
|
// folder holding no category anywhere below it is skipped; a hidden file or folder is no
|
||||||
|
// category, though a category may name a hidden rows file beside it.
|
||||||
|
func (s Source) Walk(compile func(Category) error) error {
|
||||||
|
if err := s.check(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err := s.walkDir(nil, compile)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// check refuses a data path that is empty or names no directory.
|
||||||
|
func (s Source) check() error {
|
||||||
|
if !s.onDisk {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if s.label == "" {
|
||||||
|
return fmt.Errorf("a data path is empty")
|
||||||
|
}
|
||||||
|
info, err := os.Stat(s.label)
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return noDirectory(s.label)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("data path %s: %w", s.label, err)
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
return fmt.Errorf("data path %s: not a directory", s.label)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// noDirectory is a data path naming nothing.
|
||||||
|
type noDirectory string
|
||||||
|
|
||||||
|
func (d noDirectory) Error() string { return "data path " + string(d) + ": no such directory" }
|
||||||
|
|
||||||
|
func (noDirectory) Unwrap() error { return fs.ErrNotExist }
|
||||||
|
|
||||||
|
// walkDir walks the folder that dir names, and reports whether it handed over any category.
|
||||||
|
func (s Source) walkDir(dir []string, compile func(Category) error) (bool, error) {
|
||||||
|
full := s.dirPath(dir)
|
||||||
|
entries, err := fs.ReadDir(s.fsys, full)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("%s: %w", s.labelled(full), err)
|
||||||
|
}
|
||||||
|
rows := newRowsFiles(s, full, entries)
|
||||||
|
handed := false
|
||||||
|
for _, e := range entries {
|
||||||
|
if strings.HasPrefix(e.Name(), ".") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var holds bool
|
||||||
|
if e.IsDir() {
|
||||||
|
holds, err = s.walkFolder(append(dir[:len(dir):len(dir)], e.Name()), compile)
|
||||||
|
} else {
|
||||||
|
holds, err = s.compileFile(dir, e.Name(), rows, compile)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
handed = handed || holds
|
||||||
|
}
|
||||||
|
return handed, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// walkFolder walks a subfolder, and refuses its name if it holds a category.
|
||||||
|
func (s Source) walkFolder(dir []string, compile func(Category) error) (bool, error) {
|
||||||
|
holds, err := s.walkDir(dir, compile)
|
||||||
|
if err != nil || !holds {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if err := grammar.CheckIdentifier(dir[len(dir)-1]); err != nil {
|
||||||
|
return false, fmt.Errorf("%s: folder %w", s.labelled(s.dirPath(dir)), err)
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load hands compile the category file name.json in the folder dir.
|
||||||
|
func (s Source) Load(dir []string, name string, compile func(Category) error) error {
|
||||||
|
full := s.dirPath(dir)
|
||||||
|
entries, err := fs.ReadDir(s.fsys, full)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%s: %w", s.labelled(full), err)
|
||||||
|
}
|
||||||
|
_, err = s.compileFile(dir, name+".json", newRowsFiles(s, full, entries), compile)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// compileFile hands compile a *.json file as a category named after it, and reports
|
||||||
|
// whether it was one; any other file is skipped.
|
||||||
|
func (s Source) compileFile(dir []string, file string, rows *rowsFiles, compile func(Category) error) (bool, error) {
|
||||||
|
if !strings.HasSuffix(file, ".json") {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
full := path.Join(rows.dir, file)
|
||||||
|
name := strings.TrimSuffix(file, ".json")
|
||||||
|
if err := grammar.CheckIdentifier(name); err != nil {
|
||||||
|
return false, fmt.Errorf("%s: category %w", s.labelled(full), err)
|
||||||
|
}
|
||||||
|
b, err := fs.ReadFile(s.fsys, full)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("%s: %w", s.labelled(full), err)
|
||||||
|
}
|
||||||
|
raw, err := jsonvalue.Decode(b)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("%s: %w", s.labelled(full), err)
|
||||||
|
}
|
||||||
|
if err := compile(Category{Folders: dir, Name: name, JSON: raw, rows: rows}); err != nil {
|
||||||
|
return false, fmt.Errorf("%s: %w", s.labelled(full), err)
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// rowsFiles is what a category may name beside itself: the rows files of its folder.
|
||||||
|
type rowsFiles struct {
|
||||||
|
src Source
|
||||||
|
dir string
|
||||||
|
tsv map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRowsFiles(src Source, dir string, entries []fs.DirEntry) *rowsFiles {
|
||||||
|
files := &rowsFiles{src: src, dir: dir, tsv: map[string]bool{}}
|
||||||
|
for _, e := range entries {
|
||||||
|
if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() {
|
||||||
|
files.tsv[e.Name()] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return files
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *rowsFiles) read(name string) (string, error) {
|
||||||
|
if !r.tsv[name] {
|
||||||
|
return "", fmt.Errorf("rows names %s, which is not beside it in %s", name, r.src.labelled(r.dir))
|
||||||
|
}
|
||||||
|
b, err := fs.ReadFile(r.src.fsys, path.Join(r.dir, name))
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("%s: %w", r.src.labelled(path.Join(r.dir, name)), err)
|
||||||
|
}
|
||||||
|
return string(b), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,217 @@
|
|||||||
|
package datafiles
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
)
|
||||||
|
|
||||||
|
type handed struct {
|
||||||
|
dir []string
|
||||||
|
name string
|
||||||
|
json any
|
||||||
|
}
|
||||||
|
|
||||||
|
func walk(t *testing.T, src Source) ([]handed, error) {
|
||||||
|
t.Helper()
|
||||||
|
var got []handed
|
||||||
|
err := src.Walk(func(c Category) error {
|
||||||
|
got = append(got, handed{c.Folders, c.Name, c.JSON.Any()})
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return got, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWalkHandsEachCategoryInOrder(t *testing.T) {
|
||||||
|
got, err := walk(t, Source{fsys: fstest.MapFS{
|
||||||
|
"a.json": {Data: []byte(`{"x":"1"}`)},
|
||||||
|
"sub/b.json": {Data: []byte(`"y"`)},
|
||||||
|
"sub/notes.txt": {Data: []byte("not data")},
|
||||||
|
"empty/x.txt": {Data: []byte("not data")},
|
||||||
|
".git/c.json": {Data: []byte(`"hidden"`)},
|
||||||
|
"sub/.d.json": {Data: []byte(`"hidden"`)},
|
||||||
|
"sub/deep/e.json": {Data: []byte(`"z"`)},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []handed{
|
||||||
|
{nil, "a", map[string]any{"x": "1"}},
|
||||||
|
{[]string{"sub"}, "b", "y"},
|
||||||
|
{[]string{"sub", "deep"}, "e", "z"},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("Walk handed %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadRowsReadsOnlyAFileBesideTheCategory(t *testing.T) {
|
||||||
|
src := Source{fsys: fstest.MapFS{
|
||||||
|
"t.json": {Data: []byte(`{"rows":"t.tsv"}`)},
|
||||||
|
"t.tsv": {Data: []byte("key\nA\n")},
|
||||||
|
"sub/u.tsv": {Data: []byte("key\nB\n")},
|
||||||
|
"sub/u.json": {Data: []byte(`{"rows":"u.tsv"}`)},
|
||||||
|
}, label: "lbl"}
|
||||||
|
rows := map[string]string{}
|
||||||
|
err := src.Walk(func(c Category) error {
|
||||||
|
data, err := c.ReadRows(c.Name + ".tsv")
|
||||||
|
rows[c.Name] = data
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := c.ReadRows("x.tsv"); err == nil || err.Error() != "rows names x.tsv, which is not beside it in "+map[string]string{"t": "lbl", "u": "lbl/sub"}[c.Name] {
|
||||||
|
t.Errorf("ReadRows(x.tsv) = %v, want an error saying x.tsv is not beside %s", err, c.Name)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil || rows["t"] != "key\nA\n" || rows["u"] != "key\nB\n" {
|
||||||
|
t.Errorf("Walk = %v, read %q", err, rows)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWalkIgnoresARowsFileNoCategoryNames(t *testing.T) {
|
||||||
|
if _, err := walk(t, Source{fsys: fstest.MapFS{"a.json": {Data: []byte(`"x"`)}, "a.tsv": {Data: []byte("key\n")}}, label: "lbl"}); err != nil {
|
||||||
|
t.Errorf("Walk = %v, want a.tsv ignored", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARowsFileMayBeHidden(t *testing.T) {
|
||||||
|
var rows string
|
||||||
|
err := Source{fsys: fstest.MapFS{"a.json": {Data: []byte(`"x"`)}, ".a.tsv": {Data: []byte("key\nA\n")}}}.Walk(func(c Category) error {
|
||||||
|
var err error
|
||||||
|
rows, err = c.ReadRows(".a.tsv")
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil || rows != "key\nA\n" {
|
||||||
|
t.Errorf("Walk = %v, read %q, want the hidden rows file read", err, rows)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWalkLabelsWhatACategoryFileGetsWrong(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
fs fstest.MapFS
|
||||||
|
compile error
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"its compile", fstest.MapFS{"a.json": {Data: []byte(`"x"`)}}, errors.New("bad"), "lbl/a.json: bad"},
|
||||||
|
{"its JSON", fstest.MapFS{"a.json": {Data: []byte(`{`)}}, nil, "lbl/a.json: line 1, column 1: unexpected end of JSON input"},
|
||||||
|
{"its name", fstest.MapFS{"a.b.json": {Data: []byte(`"x"`)}}, nil, "lbl/a.b.json: category "},
|
||||||
|
{"its folder's name", fstest.MapFS{"a.b/c.json": {Data: []byte(`"x"`)}}, nil, "lbl/a.b: folder "},
|
||||||
|
} {
|
||||||
|
err := Source{fsys: c.fs, label: "lbl"}.Walk(func(Category) error { return c.compile })
|
||||||
|
if err == nil || !strings.HasPrefix(err.Error(), c.want) {
|
||||||
|
t.Errorf("%s: Walk = %v, want it to start %q", c.name, err, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWalkChecksAFolderNameOnlyWhenItHoldsData(t *testing.T) {
|
||||||
|
if _, err := walk(t, Source{fsys: fstest.MapFS{"a.b/notes.txt": {Data: []byte("x")}, "c.json": {Data: []byte(`"x"`)}}}); err != nil {
|
||||||
|
t.Errorf("Walk = %v, want a folder holding no data skipped whatever its name", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWalkRefusesADiskPathThatIsNoDirectory(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
for _, c := range []struct{ path, want string }{
|
||||||
|
{"", "a data path is empty"},
|
||||||
|
{filepath.Join(dir, "none"), "data path " + filepath.Join(dir, "none") + ": no such directory"},
|
||||||
|
} {
|
||||||
|
_, err := walk(t, Dir(c.path))
|
||||||
|
if err == nil || err.Error() != c.want {
|
||||||
|
t.Errorf("Walk(%q) = %v, want %q", c.path, err, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadHandsOneCategoryAndItsRows(t *testing.T) {
|
||||||
|
src := Source{fsys: fstest.MapFS{
|
||||||
|
"sub/t.json": {Data: []byte(`{"rows":"t.tsv"}`)},
|
||||||
|
"sub/t.tsv": {Data: []byte("key\nA\n")},
|
||||||
|
"sub/u.json": {Data: []byte(`{`)},
|
||||||
|
}}
|
||||||
|
var got handed
|
||||||
|
var rows string
|
||||||
|
err := src.Load([]string{"sub"}, "t", func(c Category) error {
|
||||||
|
got = handed{c.Folders, c.Name, c.JSON.Any()}
|
||||||
|
var err error
|
||||||
|
rows, err = c.ReadRows("t.tsv")
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil || got.name != "t" || !reflect.DeepEqual(got.dir, []string{"sub"}) || rows != "key\nA\n" {
|
||||||
|
t.Errorf("Load = %v, handed %#v with rows %q", err, got, rows)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDirReadsTheDirectoryAndLabelsItsErrorsWithIt(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "a.json"), []byte(`{`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err := walk(t, Dir(dir))
|
||||||
|
if err == nil || !strings.HasPrefix(err.Error(), filepath.ToSlash(filepath.Join(dir, "a.json"))+": ") {
|
||||||
|
t.Errorf("Walk = %v, want the error labelled with the directory", err)
|
||||||
|
}
|
||||||
|
if _, err := walk(t, Dir(filepath.Join(dir, "a.json"))); err == nil || err.Error() != "data path "+filepath.Join(dir, "a.json")+": not a directory" {
|
||||||
|
t.Errorf("Walk = %v, want a file refused as no directory", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMountedDirNamesItsFilesWhereTheyAreOnDisk(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "sub"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "sub", "a.json"), []byte(`{`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err := walk(t, Dir(dir).Mounted("my.data"))
|
||||||
|
if err == nil || !strings.HasPrefix(err.Error(), filepath.ToSlash(filepath.Join(dir, "sub", "a.json"))+": ") {
|
||||||
|
t.Errorf("Walk = %v, want the error to name the file on disk", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMountedTreeListsAndReadsUnderItsMount(t *testing.T) {
|
||||||
|
m := mounted{fsys: fstest.MapFS{"a.json": {Data: []byte(`"x"`)}, "sub/b.json": {Data: []byte(`"y"`)}}, at: "my/data"}
|
||||||
|
for dir, want := range map[string][]string{".": {"my"}, "my": {"data"}, "my/data": {"a.json", "sub"}, "my/data/sub": {"b.json"}} {
|
||||||
|
entries, err := fs.ReadDir(m, dir)
|
||||||
|
var got []string
|
||||||
|
for _, e := range entries {
|
||||||
|
got = append(got, e.Name())
|
||||||
|
}
|
||||||
|
if err != nil || !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("ReadDir(%q) = %v, %v, want %v", dir, got, err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if b, err := fs.ReadFile(m, "my/data/sub/b.json"); err != nil || string(b) != `"y"` {
|
||||||
|
t.Errorf(`ReadFile("my/data/sub/b.json") = %q, %v, want "y"`, b, err)
|
||||||
|
}
|
||||||
|
if _, err := fs.ReadFile(m, "other/a.json"); !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
t.Errorf(`ReadFile("other/a.json") = %v, want it not to exist`, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMountedSourceNamesItsManifestBesideItsFiles(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
for name, c := range map[string]struct {
|
||||||
|
src Source
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
"an fs.FS": {FS(fstest.MapFS{}).Mounted("my.data"), "my/data/.fejkdata.json"},
|
||||||
|
"a data path": {Dir(dir).Mounted("my.data"), filepath.ToSlash(filepath.Join(dir, ".fejkdata.json"))},
|
||||||
|
} {
|
||||||
|
if got := c.src.ManifestPath(); got != c.want {
|
||||||
|
t.Errorf("%s: ManifestPath() = %q, want %q", name, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m, err := FS(fstest.MapFS{ManifestFile: {Data: []byte(`{"namespace": "my.data"}`)}}).Mounted("my.data").Manifest()
|
||||||
|
if err != nil || m.Namespace != "my.data" {
|
||||||
|
t.Errorf("Manifest() after Mounted = %+v, %v, want the manifest read from beside the files", m, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,246 @@
|
|||||||
|
package datafiles
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"path"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/larvit/fejkdata/internal/grammar"
|
||||||
|
"github.com/larvit/fejkdata/internal/jsonvalue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ManifestFile is the file at a source's root that describes the source.
|
||||||
|
const ManifestFile = ".fejkdata.json"
|
||||||
|
|
||||||
|
// Manifest is what a source's manifest says of it.
|
||||||
|
type Manifest struct {
|
||||||
|
// Index is every category of the source by dot path; nil where the manifest has none.
|
||||||
|
Index map[string]IndexEntry
|
||||||
|
// Namespace is the dot path the source's root sits under, "" for the namespace's root.
|
||||||
|
Namespace string
|
||||||
|
}
|
||||||
|
|
||||||
|
// IndexEntry is what an index says of a category: its parent table, "" for none, and the
|
||||||
|
// paths List advertises below it.
|
||||||
|
type IndexEntry struct {
|
||||||
|
Parent string `json:"parent,omitempty"`
|
||||||
|
Paths []string `json:"paths"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Manifest reads the manifest at the source's root, the zero Manifest where there is none.
|
||||||
|
func (s Source) Manifest() (Manifest, error) {
|
||||||
|
if err := s.check(); err != nil {
|
||||||
|
return Manifest{}, err
|
||||||
|
}
|
||||||
|
b, err := fs.ReadFile(s.fsys, path.Join(s.at, ManifestFile))
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return Manifest{}, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err)
|
||||||
|
}
|
||||||
|
m, err := decodeManifest(b)
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err)
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ManifestPath is the manifest's path as errors name it.
|
||||||
|
func (s Source) ManifestPath() string { return s.labelled(path.Join(s.at, ManifestFile)) }
|
||||||
|
|
||||||
|
// decodeManifest decodes a manifest, refusing any key or shape it does not define, and
|
||||||
|
// reports every mistake: first each entry's own, in path order, then each parent naming
|
||||||
|
// no entry.
|
||||||
|
func decodeManifest(b []byte) (Manifest, error) {
|
||||||
|
raw, err := jsonvalue.Decode(b)
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, err
|
||||||
|
}
|
||||||
|
top, isObject := raw.Any().(map[string]any)
|
||||||
|
if !isObject {
|
||||||
|
return Manifest{}, fmt.Errorf("a manifest must be an object, not %s", jsonvalue.Kind(raw.Any()))
|
||||||
|
}
|
||||||
|
var m Manifest
|
||||||
|
var errs []error
|
||||||
|
for _, k := range sortedKeys(top) {
|
||||||
|
switch k {
|
||||||
|
case "index":
|
||||||
|
var indexErrs []error
|
||||||
|
m.Index, indexErrs = decodeIndex(top[k])
|
||||||
|
errs = append(errs, indexErrs...)
|
||||||
|
case "namespace":
|
||||||
|
var err error
|
||||||
|
m.Namespace, err = decodeNamespace(top[k])
|
||||||
|
errs = append(errs, err)
|
||||||
|
default:
|
||||||
|
errs = append(errs, fmt.Errorf("unknown key %q; a manifest holds index and namespace", k))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, p := range sortedKeys(m.Index) {
|
||||||
|
if m.Namespace != "" && !strings.HasPrefix(p, m.Namespace+".") {
|
||||||
|
errs = append(errs, fmt.Errorf("index entry %q lies outside the namespace %q", p, m.Namespace))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return m, errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeNamespace(v any) (string, error) {
|
||||||
|
namespace, isString := v.(string)
|
||||||
|
if !isString {
|
||||||
|
return "", fmt.Errorf("namespace must be a string, not %s", jsonvalue.Kind(v))
|
||||||
|
}
|
||||||
|
if namespace == "" {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
if err := checkCategoryPath(namespace); err != nil {
|
||||||
|
return "", fmt.Errorf("namespace %w", err)
|
||||||
|
}
|
||||||
|
return namespace, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeIndex(v any) (map[string]IndexEntry, []error) {
|
||||||
|
entries, isObject := v.(map[string]any)
|
||||||
|
if !isObject {
|
||||||
|
return nil, []error{fmt.Errorf("index must be an object, not %s", jsonvalue.Kind(v))}
|
||||||
|
}
|
||||||
|
index := map[string]IndexEntry{}
|
||||||
|
var errs []error
|
||||||
|
for _, p := range sortedKeys(entries) {
|
||||||
|
e, err := decodeEntry(p, entries[p])
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, fmt.Errorf("index entry %q: %w", p, err))
|
||||||
|
}
|
||||||
|
index[p] = e
|
||||||
|
}
|
||||||
|
for _, p := range sortedKeys(index) {
|
||||||
|
for _, err := range []error{checkOutside(index, p), checkParent(index, p)} {
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, fmt.Errorf("index entry %q: %w", p, err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return index, errs
|
||||||
|
}
|
||||||
|
|
||||||
|
// decodeEntry decodes one entry, leaving out a parent it refuses, so checkParent names no
|
||||||
|
// mistake twice.
|
||||||
|
func decodeEntry(p string, v any) (IndexEntry, error) {
|
||||||
|
var e IndexEntry
|
||||||
|
errs := []error{checkCategoryPath(p)}
|
||||||
|
fields, isObject := v.(map[string]any)
|
||||||
|
if !isObject {
|
||||||
|
errs = append(errs, fmt.Errorf("an entry must be an object, not %s", jsonvalue.Kind(v)))
|
||||||
|
return e, errors.Join(errs...)
|
||||||
|
}
|
||||||
|
if _, has := fields["paths"]; !has {
|
||||||
|
errs = append(errs, fmt.Errorf("paths is missing"))
|
||||||
|
}
|
||||||
|
for _, k := range sortedKeys(fields) {
|
||||||
|
switch k {
|
||||||
|
case "parent":
|
||||||
|
parent, err := decodeParent(fields[k])
|
||||||
|
e.Parent = parent
|
||||||
|
errs = append(errs, err)
|
||||||
|
case "paths":
|
||||||
|
var err error
|
||||||
|
e.Paths, err = decodePaths(fields[k])
|
||||||
|
errs = append(errs, err)
|
||||||
|
default:
|
||||||
|
errs = append(errs, fmt.Errorf("unknown key %q; an entry holds parent and paths", k))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return e, errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeParent(v any) (string, error) {
|
||||||
|
parent, isString := v.(string)
|
||||||
|
if !isString {
|
||||||
|
return "", fmt.Errorf("parent must be a string, not %s", jsonvalue.Kind(v))
|
||||||
|
}
|
||||||
|
if err := grammar.CheckIdentifier(parent); err != nil {
|
||||||
|
return "", fmt.Errorf("parent %w", err)
|
||||||
|
}
|
||||||
|
return parent, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// decodePaths decodes a list of paths below a category, "" being the category itself.
|
||||||
|
func decodePaths(v any) ([]string, error) {
|
||||||
|
items, isList := v.([]any)
|
||||||
|
if !isList {
|
||||||
|
return nil, fmt.Errorf("paths must be a list, not %s", jsonvalue.Kind(v))
|
||||||
|
}
|
||||||
|
out := make([]string, 0, len(items))
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var errs []error
|
||||||
|
for i, item := range items {
|
||||||
|
q, isString := item.(string)
|
||||||
|
if !isString {
|
||||||
|
errs = append(errs, fmt.Errorf("paths item %d must be a string, not %s", i+1, jsonvalue.Kind(item)))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if q != "" {
|
||||||
|
if err := checkCategoryPath(q); err != nil {
|
||||||
|
errs = append(errs, fmt.Errorf("paths item %d: %w", i+1, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if seen[q] {
|
||||||
|
errs = append(errs, fmt.Errorf("paths item %d repeats %q", i+1, q))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[q] = true
|
||||||
|
out = append(out, q)
|
||||||
|
}
|
||||||
|
return out, errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkOutside refuses an entry whose path passes through another entry's category.
|
||||||
|
func checkOutside(index map[string]IndexEntry, p string) error {
|
||||||
|
segs := strings.Split(p, ".")
|
||||||
|
for n := 1; n < len(segs); n++ {
|
||||||
|
outer := strings.Join(segs[:n], ".")
|
||||||
|
if _, isEntry := index[outer]; isEntry {
|
||||||
|
return fmt.Errorf("%q is a category, so it holds no other", outer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkParent refuses a parent naming no entry beside p: a table loads with its parent,
|
||||||
|
// from one source.
|
||||||
|
func checkParent(index map[string]IndexEntry, p string) error {
|
||||||
|
parent := index[p].Parent
|
||||||
|
if parent == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
sibling := parent
|
||||||
|
if dot := strings.LastIndexByte(p, '.'); dot >= 0 {
|
||||||
|
sibling = p[:dot+1] + parent
|
||||||
|
}
|
||||||
|
if _, ok := index[sibling]; !ok {
|
||||||
|
return fmt.Errorf("parent %q names no entry in this entry's folder; a table in an indexed source links only to a parent in the same index, so to link under another source's table, drop this index, and the source loads whole at start", parent)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkCategoryPath(p string) error {
|
||||||
|
for _, seg := range strings.Split(p, ".") {
|
||||||
|
if err := grammar.CheckIdentifier(seg); err != nil {
|
||||||
|
return fmt.Errorf("%q: %w", p, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedKeys[V any](m map[string]V) []string {
|
||||||
|
keys := make([]string, 0, len(m))
|
||||||
|
for k := range m {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
return keys
|
||||||
|
}
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
package datafiles
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mounted is fsys with its root moved down to the folder at, so a file f of fsys is at/f. It
|
||||||
|
// serves what a walk asks of a tree, a folder's entries and a file's bytes: the folders above
|
||||||
|
// at list only the next folder towards it, and opening one fails.
|
||||||
|
type mounted struct {
|
||||||
|
fsys fs.FS
|
||||||
|
at string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m mounted) Open(name string) (fs.File, error) {
|
||||||
|
if inner, below := m.inner(name); below {
|
||||||
|
return m.fsys.Open(inner)
|
||||||
|
}
|
||||||
|
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m mounted) ReadFile(name string) ([]byte, error) {
|
||||||
|
if inner, below := m.inner(name); below {
|
||||||
|
return fs.ReadFile(m.fsys, inner)
|
||||||
|
}
|
||||||
|
return nil, &fs.PathError{Op: "read", Path: name, Err: fs.ErrNotExist}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m mounted) ReadDir(name string) ([]fs.DirEntry, error) {
|
||||||
|
if inner, below := m.inner(name); below {
|
||||||
|
return fs.ReadDir(m.fsys, inner)
|
||||||
|
}
|
||||||
|
rest, above := strings.CutPrefix(m.at, name+"/")
|
||||||
|
if name == "." {
|
||||||
|
rest, above = m.at, true
|
||||||
|
}
|
||||||
|
if !above {
|
||||||
|
return nil, &fs.PathError{Op: "readdir", Path: name, Err: fs.ErrNotExist}
|
||||||
|
}
|
||||||
|
next, _, _ := strings.Cut(rest, "/")
|
||||||
|
return []fs.DirEntry{folderEntry(next)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// inner is name as fsys spells it, where name lies at or below at.
|
||||||
|
func (m mounted) inner(name string) (string, bool) {
|
||||||
|
inner, below := underMount(m.at, name)
|
||||||
|
return inner, below && fs.ValidPath(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// underMount is name as the tree mounted at at spells it, where name lies at or below at.
|
||||||
|
func underMount(at, name string) (string, bool) {
|
||||||
|
switch name {
|
||||||
|
case at:
|
||||||
|
return ".", true
|
||||||
|
case "":
|
||||||
|
return name, false
|
||||||
|
}
|
||||||
|
if at == "" {
|
||||||
|
return name, true
|
||||||
|
}
|
||||||
|
return strings.CutPrefix(name, at+"/")
|
||||||
|
}
|
||||||
|
|
||||||
|
// folderEntry is a folder above the mount, as a folder's entries list it.
|
||||||
|
type folderEntry string
|
||||||
|
|
||||||
|
func (e folderEntry) Name() string { return string(e) }
|
||||||
|
func (e folderEntry) IsDir() bool { return true }
|
||||||
|
func (e folderEntry) Type() fs.FileMode { return fs.ModeDir }
|
||||||
|
func (e folderEntry) Info() (fs.FileInfo, error) {
|
||||||
|
return nil, &fs.PathError{Op: "stat", Path: string(e), Err: fs.ErrNotExist}
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
// Package datatype names what a record column holds.
|
||||||
|
package datatype
|
||||||
|
|
||||||
|
import "fmt"
|
||||||
|
|
||||||
|
// DataType is what a record column holds, which decides how a record writes its value.
|
||||||
|
type DataType int
|
||||||
|
|
||||||
|
// The datatypes a column declares with "datatype"; a column without one is a string.
|
||||||
|
const (
|
||||||
|
String DataType = iota
|
||||||
|
Integer
|
||||||
|
Number
|
||||||
|
Boolean
|
||||||
|
)
|
||||||
|
|
||||||
|
// Count is the number of datatypes, for an array indexed by one.
|
||||||
|
const Count = Boolean + 1
|
||||||
|
|
||||||
|
var (
|
||||||
|
names = [Count]string{"string", "integer", "number", "boolean"}
|
||||||
|
nouns = [Count]string{"text", "an integer", "a number", "a boolean"}
|
||||||
|
)
|
||||||
|
|
||||||
|
// String is the datatype as data spells it.
|
||||||
|
func (d DataType) String() string {
|
||||||
|
if d < 0 || d >= Count {
|
||||||
|
return fmt.Sprintf("DataType(%d)", int(d))
|
||||||
|
}
|
||||||
|
return names[d]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Noun is the datatype as an error names what a value is.
|
||||||
|
func Noun(d DataType) string {
|
||||||
|
if d < 0 || d >= Count {
|
||||||
|
return d.String()
|
||||||
|
}
|
||||||
|
return nouns[d]
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
package datatype
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestEachDatatypeIsSpelledAsDataWritesIt(t *testing.T) {
|
||||||
|
for d, want := range map[DataType][2]string{
|
||||||
|
String: {"string", "text"},
|
||||||
|
Integer: {"integer", "an integer"},
|
||||||
|
Number: {"number", "a number"},
|
||||||
|
Boolean: {"boolean", "a boolean"},
|
||||||
|
} {
|
||||||
|
if d.String() != want[0] || Noun(d) != want[1] {
|
||||||
|
t.Errorf("%d: String = %q, Noun = %q, want %q", int(d), d.String(), Noun(d), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if Count != 4 {
|
||||||
|
t.Errorf("Count = %d, want the four datatypes", Count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStringNamesADatatypeNoDataWrites(t *testing.T) {
|
||||||
|
if got := DataType(9).String(); got != "DataType(9)" {
|
||||||
|
t.Errorf("String = %q, want DataType(9)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNounNamesADatatypeNoDataWrites(t *testing.T) {
|
||||||
|
if got := Noun(DataType(9)); got != "DataType(9)" {
|
||||||
|
t.Errorf("Noun = %q, want DataType(9)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
// Package drawstate holds a generator's seeded randomness and its {seq()} counters.
|
||||||
|
package drawstate
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math/rand/v2"
|
||||||
|
"sort"
|
||||||
|
)
|
||||||
|
|
||||||
|
// State is one generator's draw state, kept across its renders.
|
||||||
|
type State struct {
|
||||||
|
rand *rand.Rand
|
||||||
|
counters map[string]uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// New builds the state a seed reproduces.
|
||||||
|
func New(seed uint64) *State {
|
||||||
|
return &State{rand: rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15)), counters: map[string]uint64{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *State) IntN(n int) int { return s.rand.IntN(n) }
|
||||||
|
|
||||||
|
func (s *State) Float64() float64 { return s.rand.Float64() }
|
||||||
|
|
||||||
|
// Weighted draws an index into cum, a running sum of weights of 0 or more with a positive total,
|
||||||
|
// with odds by weight, so an index of weight 0 is never drawn.
|
||||||
|
func (s *State) Weighted(cum []float64) int {
|
||||||
|
total := cum[len(cum)-1]
|
||||||
|
x := s.Float64() * total
|
||||||
|
// x stays below total: a product of a float64 under 1 and a positive float64 rounds below it.
|
||||||
|
return sort.Search(len(cum), func(i int) bool { return cum[i] > x })
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seq advances the counter {seq(key)} reads, or {seq()} for an empty key, and returns it.
|
||||||
|
func (s *State) Seq(key string) uint64 {
|
||||||
|
s.counters[key]++
|
||||||
|
return s.counters[key]
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user