From b0db82f2f11c445b00729042c7f2f4788983943d Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 18 Sep 2026 23:02:45 +0200 Subject: [PATCH] Rename misc.country to misc.territory, name each one's sovereign state, and state the four data rules that settle it --- CHANGELOG.md | 12 +- DATA-LICENSES.md | 2 +- README.md | 69 ++++-- cmd/fejkdata/main.go | 2 +- data-import/territory.py | 25 +- data/misc/territory.json | 2 +- data/misc/territory.tsv | 484 +++++++++++++++++++-------------------- todo.md | 16 +- 8 files changed, 336 insertions(+), 276 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 357feb2..7815d75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ replacement, and each removed path, column or flag. - First release: the CLI, the library and the shipped data set. - Table categories: a category JSON naming a `rows` TSV beside it, with the options `key`, `name`, `weight` and `parent`; a path selects a row by key or name, - `misc.country[SE]`, and descends to a linked table by name; linked tables draw + `misc.territory[SE]`, and descends to a linked table by name; linked tables draw consistently within one render and draw group. `rows` is an option, so no template may carry a field of that name. A `name` without a `key` resolves inside the table's `parent`. Refused at `New`: a `name` without a `key` or a @@ -21,10 +21,12 @@ replacement, and each removed path, column or flag. row of, or two paths pinning different rows of one table. - `New` refuses a root choice of templates sharing one format and one set of string fields, naming the rows TSV to write instead. -- `misc.country`, `misc.currency`, `misc.language`, `misc.httpstatus` and - `misc.mimetype` are tables. `misc.country` is every ISO 3166 country that has a - capital, a currency and a TLD, with the columns `calling-code`, `capital`, - `currency`, `flag`, `languages`, `numeric` and `tld` added; `misc.currency` the +- `misc.territory`, `misc.currency`, `misc.language`, `misc.httpstatus` and + `misc.mimetype` are tables. `misc.territory` is every ISO 3166-1 territory that has + a capital, a currency and a TLD, with the columns `calling-code`, `capital`, + `country`, `currency`, `flag`, `languages`, `numeric` and `tld` added, `country` + naming the sovereign state it belongs to and itself where it is one; + `misc.currency` the current ISO 4217 currencies with a minor unit, with `decimals` and `numeric` added, and its symbols from CLDR; `misc.language` every ISO 639-2 entry carrying a 639-1 code, with a `code3` column holding its 639-2/T code; `misc.httpstatus` each diff --git a/DATA-LICENSES.md b/DATA-LICENSES.md index 2ff85a6..1a43fb0 100644 --- a/DATA-LICENSES.md +++ b/DATA-LICENSES.md @@ -15,7 +15,7 @@ Every shipped dataset, its source, its licence and the attribution it asks for. | `en_US/first-name.tsv` | [SSA](https://www.ssa.gov/oact/babynames/) baby names, births 1930 to 2020, through [hackerb9/ssa-baby-names](https://github.com/hackerb9/ssa-baby-names) | public domain | none required | `data-import/names-us.py` | | `en_US/last-name.tsv` | Census Bureau surnames occurring 100 or more times, 2010 | public domain | none required | `data-import/names-us.py` | | `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv` | curated (Skatteverket's test birth numbers are facts) | — | — | — | -| `misc/country.tsv` | [datasets/country-codes](https://github.com/datasets/country-codes) | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/country.py` | +| `misc/territory.tsv` | [datasets/country-codes](https://github.com/datasets/country-codes) | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/territory.py` | | `misc/currency.tsv` | [datasets/currency-codes](https://github.com/datasets/currency-codes); symbols from [Unicode CLDR](https://github.com/unicode-org/cldr) `en.xml` and `root.xml` | PDDL 1.0; [Unicode License v3](https://www.unicode.org/license.txt) | CLDR: "Copyright © 1991-2025 Unicode, Inc. Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries." | `data-import/currency.py` | | `misc/httpstatus.tsv` | [IANA HTTP Status Code Registry](https://www.iana.org/assignments/http-status-codes/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpstatus.py` | | `misc/language.tsv` | [datasets/language-codes](https://github.com/datasets/language-codes), the [Library of Congress](https://www.loc.gov/standards/iso639-2/) ISO 639-2 register | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/language.py` | diff --git a/README.md b/README.md index ad915dc..e532d17 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ fejkdata sv_SE.person.last # Eriksson fejkdata --seed 42 sv_SE.address # the same address every run fejkdata -n 3 --separator ', ' sv_SE.word # nät, barn, sol fejkdata --list # every path the data offers -fejkdata 'misc.country[SE].capital' # Stockholm — a table's row, selected by key or name -fejkdata 'geo.SE.locality[Lund].street' # Fjelievägen — a linked table, drawn inside the row +fejkdata 'misc.territory[SE].capital' # Stockholm — a table's row, selected by key or name +fejkdata 'geo.SE.locality[Lund].street' # Fjelievägen — a linked table, drawn inside the row fejkdata --data-path ./mydata sv_SE.word # layer a directory over the shipped data fejkdata --no-shipped-data -d ./mydata --list # only your data fejkdata 'name: {/sv_SE.person.last}' # name: — an inline template @@ -161,15 +161,20 @@ Each locale carries `address`, `color`, `company`, `date`, `email`, `first-name` `ip`, `last-name`, `person`, `phone`, `price`, `sentence`, `sex`, `time`, `url`, `username`, `version` and `word`, formatted per locale; `sv_SE` adds `personnummer` and `samordningsnummer`, `en_US` adds `ssn` and `itin`. `misc` -carries `car`, `coordinate`, `country` (ISO 3166), `creditcard` (Luhn-valid), -`currency` (ISO 4217), `datetime` (RFC 3339), `emoji`, `httpstatus`, `language` -(ISO 639-1, with its 639-2/T code), `mac`, `mimetype`, `objectid`, `timezone` +carries `car`, `coordinate`, `creditcard` (Luhn-valid), `currency` (ISO 4217), +`datetime` (RFC 3339), `emoji`, `httpstatus`, `language` (ISO 639-1, with its +639-2/T code), `mac`, `mimetype`, `objectid`, `territory` (ISO 3166-1), `timezone` (IANA), `useragent` and `uuid` (v4). Many carry sub-fields — `misc.currency.symbol`, -`misc.country.alpha2`, `misc.httpstatus.code` — which `--list` shows. `country`, -`currency`, `httpstatus`, `language` and `mimetype` are [tables](#table), so -`misc.country[SE].capital` and `misc.currency[Euro].symbol` select a row; +`misc.territory.alpha2`, `misc.httpstatus.code` — which `--list` shows. `currency`, +`httpstatus`, `language`, `mimetype` and `territory` are [tables](#table), so +`misc.territory[SE].capital` and `misc.currency[Euro].symbol` select a row; [`DATA-LICENSES.md`](DATA-LICENSES.md) names each table's source and licence. +ISO 3166-1 codes territories, not sovereign states, so that is what the table is +called: Greenland and Åland have codes of their own, and `misc.territory.country` +names the state each belongs to — `DK` for Greenland, `FI` for Åland, and its own +code for a sovereign one. + `sex`, `first-name` and `last-name` are tables weighted by bearers, from SCB, the SSA and the Census Bureau. `first-name` links to `sex`, so `sv_SE.sex[f].first-name` draws a woman's name, and a name both sexes carry is a row under each, so @@ -447,11 +452,11 @@ inline template has no file beside it, so there it stays a choice. ### Row selection -`[key]` or `[name]` after a table's name selects one row: `misc.country[SE]` and -`misc.country[Sweden]` name one row, and `misc.country[SE].capital` reads its column. +`[key]` or `[name]` after a table's name selects one row: `misc.territory[SE]` and +`misc.territory[Sweden]` name one row, and `misc.territory[SE].capital` reads its column. A name naming several rows is an error listing their keys, unless a row selected before it settles which ([Linked tables](#linked-tables)). A selector is part of the path, so it works -wherever a path does: `Fake`, `FakeRecord`, a `{/misc.country[SE].capital}` reference +wherever a path does: `Fake`, `FakeRecord`, a `{/misc.territory[SE].capital}` reference and a struct tag. A dot inside the brackets belongs to the key or name, so `city[St. Louis]` selects it. A path starts with a name, and `[` still opens a JSON array at the start of a CLI argument, so `'[SE]'` alone names nothing. @@ -772,6 +777,13 @@ App developers writing tests and fixtures, in Go and at a shell: 9. **Fast enough to be free** — a value renders in about a microsecond and `New` parses and validates the whole set once upfront, so generating fixtures stays noise against a test's own runtime. +10. **Data is sourced, or on its way there** — a shipped fact, a name, place, + code, id or classification, is read from a register or open dataset by a + [`data-import/`](data-import) script wherever one exists to read; where none + does yet a small hand-written set ships and [`todo.md`](todo.md) carries the + step that replaces it. Only non-factual copy stays authored. A sourced table + holds the rows its source holds: none is added by hand, and one is dropped + only by a rule the script states. ## Decisions @@ -1124,11 +1136,40 @@ App developers writing tests and fixtures, in Go and at a shell: tells the two apart, `sex[f].first-name[Kim]`, the ambiguity error spells each row inside its parent, and a name repeating inside one parent row is refused at load, since nothing could then select it. -- **`misc.country` carries a currency code, it does not link to `misc.currency`.** +- **`misc` is what every locale shares.** A category whose facts differ by country + belongs in that country's locale, read from the register that country's own + records use; `misc` takes only sources that are international. So NHTSA vPIC + builds `en_US.car` and Mobility Sweden's registrations `sv_SE.car`, never + `misc.car`. +- **A register's canonical spelling loses to the one its domain writes.** Where a + source offers several spellings of one fact, the shipped one is what records in + that domain carry. `misc.timezone` reads `zone.tab` and not the `zone1970.tab` + that supersedes it, because the latter keeps one zone per set of countries that + have agreed since 1970: it spells Sweden `Europe/Berlin`, and no Swedish system + writes that. For the same reason `misc.language` takes the ISO 639-2 register's + first synonym over CLDR, which says "Chinese, Mandarin" for `zh`. +- **`misc.territory` is the spine, and a `misc` table naming a territory links to + it.** Where every territory row has a child the column is a `parent`, and the + import drops the child rows whose territory the set does not ship — 17 of + `misc.timezone`'s, Antarctica's ten among them. Agreement across a record is + worth more than the last rows of a table. Where no such link can hold the fact + stays a column. +- **`misc.territory` names its sovereign in a column, and there is no `misc.country` + table.** ISO 3166-1 codes territories, so `territory` is the honest name, and + `is_independent` in the register gives each one's state. A second table of the 195 + sovereigns would hold a `DK` row beside the territory `DK` row, both carrying + Denmark's capital, currency, flag and TLD — two owners for one fact, drifting at + the next import. Splitting the columns to avoid that is worse: put `capital` on + the territory alone and a country row can no longer name Copenhagen. The column + cannot be a `parent`, since a table never reaches its own family; a test proves + every value names a row instead. A territory the register records no state for + stands alone, which is `EH` alone, and naming one for it would be a claim + fejkdata has no business making. +- **`misc.territory` carries a currency code, it does not link to `misc.currency`.** A `parent` demands a child for every parent row, and ISO 4217 registers codes no country's row can name: the funds codes (Mvdol, WIR Euro, US Dollar (Next day)), and VED beside VES, both Venezuela's, of which a country row names one. Linking - would trade the register for the link, and `misc.country.currency` already pairs a + would trade the register for the link, and `misc.territory.currency` already pairs a country with its currency in one draw. - **An extension may name two media types.** `.xml`, `.rtf`, `.sub`, `.mpp` and `.ac` each name two rows of `misc.mimetype`. Separating them would mean dropping a @@ -1207,7 +1248,7 @@ and `names-us.py` rejects a client for a while after a burst, so `--surnames` ta a copy of the surname file: ```sh -docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/country.py +docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/territory.py docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/currency.py docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/geo-us.py docker compose run --rm --user "$(id -u):$(id -g)" -e TRAFIKVERKET_API_KEY data-import data-import/geo-se.py diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index 57ccd79..327e5c3 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -24,7 +24,7 @@ import ( const usage = `Usage: fejkdata [flags] a category, or a dotted path into one (person, person.last); - a table's row by key or name: 'misc.country[SE]', 'misc.country[Sweden].tld' + a table's row by key or name: 'misc.territory[SE]', 'misc.territory[Sweden].tld'