From e9d86036117f0dca6febc8ee62cfb0adcd0ae945 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 18 Sep 2026 23:02:43 +0200 Subject: [PATCH 01/14] Expect misc.territory and the sovereign it names, in place of misc.country --- cmd/fejkdata/main_test.go | 4 +- data-import/{country.py => territory.py} | 0 data/misc/{country.json => territory.json} | 0 data/misc/{country.tsv => territory.tsv} | 0 data_test.go | 35 +++--- inline_test.go | 6 +- loading_test.go | 2 +- table_test.go | 119 +++++++++++++-------- 8 files changed, 98 insertions(+), 68 deletions(-) rename data-import/{country.py => territory.py} (100%) rename data/misc/{country.json => territory.json} (100%) rename data/misc/{country.tsv => territory.tsv} (100%) diff --git a/cmd/fejkdata/main_test.go b/cmd/fejkdata/main_test.go index e11ec89..1924d31 100644 --- a/cmd/fejkdata/main_test.go +++ b/cmd/fejkdata/main_test.go @@ -352,8 +352,8 @@ func TestRunSelectsATableRow(t *testing.T) { if code, _, errb := runOut("--no-shipped-data", "-d", dir, "[Skรฅne lรคn]"); code != 2 || !strings.Contains(errb, "JSON") { t.Fatalf("[Skรฅne lรคn] = %d, stderr=%q, want misuse: a leading bracket that is no JSON names nothing", code, errb) } - if code, out, _ := runOut("--seed", "1", "--format", "csv", "misc.country[SE]"); code != 0 || !strings.HasPrefix(out, "alpha2,") || !strings.Contains(out, "\nSE,SWE,") { - t.Fatalf("--format csv misc.country[SE] = %d, %q", code, out) + if code, out, _ := runOut("--seed", "1", "--format", "csv", "misc.territory[SE]"); code != 0 || !strings.HasPrefix(out, "alpha2,") || !strings.Contains(out, "\nSE,SWE,") { + t.Fatalf("--format csv misc.territory[SE] = %d, %q", code, out) } } diff --git a/data-import/country.py b/data-import/territory.py similarity index 100% rename from data-import/country.py rename to data-import/territory.py diff --git a/data/misc/country.json b/data/misc/territory.json similarity index 100% rename from data/misc/country.json rename to data/misc/territory.json diff --git a/data/misc/country.tsv b/data/misc/territory.tsv similarity index 100% rename from data/misc/country.tsv rename to data/misc/territory.tsv diff --git a/data_test.go b/data_test.go index 3583400..b477724 100644 --- a/data_test.go +++ b/data_test.go @@ -95,23 +95,24 @@ func TestShippedMiscCategories(t *testing.T) { func TestShippedMiscReferenceData(t *testing.T) { f := newGenerator(t, "data/misc", WithSeed(1)) re := map[string]*regexp.Regexp{ - "currency": regexp.MustCompile(`^[A-Z]{3}$`), - "currency.name": regexp.MustCompile(`\p{L}`), - "currency.symbol": regexp.MustCompile(`^\S+$`), - "country": regexp.MustCompile(`\p{L}`), - "country.alpha2": regexp.MustCompile(`^[A-Z]{2}$`), - "country.alpha3": regexp.MustCompile(`^[A-Z]{3}$`), - "language": regexp.MustCompile(`\p{L}`), - "language.code": regexp.MustCompile(`^[a-z]{2}$`), - "language.code3": regexp.MustCompile(`^[a-z]{3}$`), - "timezone": regexp.MustCompile(`^([A-Za-z_]+(/[A-Za-z_]+)+|UTC)$`), - "mimetype": regexp.MustCompile(`^[a-z]+/[a-z0-9.+-]+$`), - "mimetype.ext": regexp.MustCompile(`^\.[a-z0-9_-]+$`), - "httpstatus": regexp.MustCompile(`^[1-5]\d{2} \S.*$`), - "httpstatus.code": regexp.MustCompile(`^[1-5]\d{2}$`), - "useragent": regexp.MustCompile(`^Mozilla/5\.0 .+`), - "car": regexp.MustCompile(`^\S.* \S`), - "car.maker": regexp.MustCompile(`^\S`), + "currency": regexp.MustCompile(`^[A-Z]{3}$`), + "currency.name": regexp.MustCompile(`\p{L}`), + "currency.symbol": regexp.MustCompile(`^\S+$`), + "territory": regexp.MustCompile(`\p{L}`), + "territory.alpha2": regexp.MustCompile(`^[A-Z]{2}$`), + "territory.alpha3": regexp.MustCompile(`^[A-Z]{3}$`), + "territory.country": regexp.MustCompile(`^[A-Z]{2}$`), + "language": regexp.MustCompile(`\p{L}`), + "language.code": regexp.MustCompile(`^[a-z]{2}$`), + "language.code3": regexp.MustCompile(`^[a-z]{3}$`), + "timezone": regexp.MustCompile(`^([A-Za-z_]+(/[A-Za-z_]+)+|UTC)$`), + "mimetype": regexp.MustCompile(`^[a-z]+/[a-z0-9.+-]+$`), + "mimetype.ext": regexp.MustCompile(`^\.[a-z0-9_-]+$`), + "httpstatus": regexp.MustCompile(`^[1-5]\d{2} \S.*$`), + "httpstatus.code": regexp.MustCompile(`^[1-5]\d{2}$`), + "useragent": regexp.MustCompile(`^Mozilla/5\.0 .+`), + "car": regexp.MustCompile(`^\S.* \S`), + "car.maker": regexp.MustCompile(`^\S`), } for i := 0; i < 100; i++ { for p, rx := range re { diff --git a/inline_test.go b/inline_test.go index 3e47467..c2876d5 100644 --- a/inline_test.go +++ b/inline_test.go @@ -108,9 +108,9 @@ func TestFakeTemplateErrors(t *testing.T) { {`name: {/no.such.path}`, "no entry"}, {`name: {..nope}`, "write {/nope}"}, {`{"format":"x"}`, "is a string"}, - {`{/misc.country} {/misc.country.alpha2}`, "renders a level that {/misc.country.alpha2} reads a path into; name the fields you want instead, or draw them apart with a drawGroup"}, - {`{"format":"{/misc.country.alpha2} {x}","x":"{/misc.country}"}`, "reads a path into"}, - {`{"format":"{/misc.country.alpha2}","drawGroup":"g"}`, "nothing can reference"}, + {`{/misc.territory} {/misc.territory.alpha2}`, "renders a level that {/misc.territory.alpha2} reads a path into; name the fields you want instead, or draw them apart with a drawGroup"}, + {`{"format":"{/misc.territory.alpha2} {x}","x":"{/misc.territory}"}`, "reads a path into"}, + {`{"format":"{/misc.territory.alpha2}","drawGroup":"g"}`, "nothing can reference"}, } { _, err := f.FakeTemplate(c.input) if err == nil || !strings.Contains(err.Error(), c.want) { diff --git a/loading_test.go b/loading_test.go index e335a64..2e0e594 100644 --- a/loading_test.go +++ b/loading_test.go @@ -154,7 +154,7 @@ func TestListedPathsAllRender(t *testing.T) { } } } - for _, p := range []string{"misc.car.maker", "misc.country.alpha2", "misc.country.numeric", "misc.currency.symbol", "misc.httpstatus.code", "misc.mimetype.ext"} { + for _, p := range []string{"misc.car.maker", "misc.territory.alpha2", "misc.territory.numeric", "misc.currency.symbol", "misc.httpstatus.code", "misc.mimetype.ext"} { if !slices.Contains(paths, p) { t.Errorf("List() omits %q, which the README advertises and Fake renders", p) } diff --git a/table_test.go b/table_test.go index 018a40d..a07f462 100644 --- a/table_test.go +++ b/table_test.go @@ -644,17 +644,17 @@ func TestTableRowsAreAlternatives(t *testing.T) { func TestTableOverridesByLayering(t *testing.T) { mine := writeFiles(t, map[string]string{ - "misc/country.json": `{"format":"{name}","rows":"country.tsv","key":"alpha2"}`, - "misc/country.tsv": "alpha2\tname\nXX\tNowhere\nYY\tElsewhere\n", + "misc/territory.json": `{"format":"{name}","rows":"territory.tsv","key":"alpha2"}`, + "misc/territory.tsv": "alpha2\tname\nXX\tNowhere\nYY\tElsewhere\n", }) f, err := New(WithDataPath(mine), WithSeed(1)) if err != nil { t.Fatal(err) } - if v := fake(t, f, "misc.country[XX]"); v != "Nowhere" { - t.Fatalf("misc.country[XX] = %q, want the layered table", v) + if v := fake(t, f, "misc.territory[XX]"); v != "Nowhere" { + t.Fatalf("misc.territory[XX] = %q, want the layered table", v) } - if paths := f.List(); slices.Contains(paths, "misc.country.alpha3") { + if paths := f.List(); slices.Contains(paths, "misc.territory.alpha3") { t.Fatal("List() still offers the shipped table's columns under an overridden category") } } @@ -662,50 +662,55 @@ func TestTableOverridesByLayering(t *testing.T) { func TestShippedTables(t *testing.T) { f := newGenerator(t, "data/misc", WithSeed(1)) for path, want := range map[string]string{ - "country[SE]": "Sweden", - "country[Sweden].alpha3": "SWE", - "country[SE].numeric": "752", - "country[SE].tld": ".se", - "country[SE].calling-code": "46", - "country[SE].capital": "Stockholm", - "country[SE].currency": "SEK", - "country[SE].flag": "๐Ÿ‡ธ๐Ÿ‡ช", - "currency[SEK].name": "Swedish Krona", - "currency[SEK].symbol": "kr", - "currency[SEK].numeric": "752", - "currency[SEK].decimals": "2", - "currency[Euro].code": "EUR", - "language[sv]": "Swedish", - "language[sv].code3": "swe", - "language[Swedish].code": "sv", - "language[nl]": "Dutch", - "httpstatus[200]": "200 OK", - "httpstatus[404]": "404 Not Found", - "httpstatus[404].reason": "Not Found", - "httpstatus[451].reason": "Unavailable For Legal Reasons", - "httpstatus[500].reason": "Internal Server Error", - "mimetype[application/json].ext": ".json", - "mimetype[text/markdown].ext": ".md", - "mimetype[.jpg]": "image/jpeg", - "mimetype[.mov]": "video/quicktime", - "mimetype[.mp3]": "audio/mpeg", - "mimetype[.mp4]": "video/mp4", - "mimetype[.ogg]": "audio/ogg", + "territory[SE]": "Sweden", + "territory[Sweden].alpha3": "SWE", + "territory[SE].numeric": "752", + "territory[SE].tld": ".se", + "territory[SE].calling-code": "46", + "territory[SE].capital": "Stockholm", + "territory[SE].currency": "SEK", + "territory[SE].flag": "๐Ÿ‡ธ๐Ÿ‡ช", + "territory[SE].country": "SE", + "territory[GL]": "Greenland", + "territory[GL].country": "DK", + "territory[ร…land Islands].country": "FI", + "currency[SEK].name": "Swedish Krona", + "currency[SEK].symbol": "kr", + "currency[SEK].numeric": "752", + "currency[SEK].decimals": "2", + "currency[Euro].code": "EUR", + "language[sv]": "Swedish", + "language[sv].code3": "swe", + "language[Swedish].code": "sv", + "language[nl]": "Dutch", + "httpstatus[200]": "200 OK", + "httpstatus[404]": "404 Not Found", + "httpstatus[404].reason": "Not Found", + "httpstatus[451].reason": "Unavailable For Legal Reasons", + "httpstatus[500].reason": "Internal Server Error", + "mimetype[application/json].ext": ".json", + "mimetype[text/markdown].ext": ".md", + "mimetype[.jpg]": "image/jpeg", + "mimetype[.mov]": "video/quicktime", + "mimetype[.mp3]": "audio/mpeg", + "mimetype[.mp4]": "video/mp4", + "mimetype[.ogg]": "audio/ogg", } { if got := fake(t, f, path); got != want { t.Errorf("Fake(%q) = %q, want %q", path, got, want) } } re := map[string]*regexp.Regexp{ - "country.numeric": regexp.MustCompile(`^\d{3}$`), - "country.tld": regexp.MustCompile(`^\.[a-z]{2}$`), - "country.calling-code": regexp.MustCompile(`^\d{1,4}(-\d{3})?$`), - "country.capital": regexp.MustCompile(`\p{L}`), - "country.currency": regexp.MustCompile(`^[A-Z]{3}$`), - "country.flag": regexp.MustCompile(`^[\x{1F1E6}-\x{1F1FF}]{2}$`), - "country.languages": regexp.MustCompile(`^[a-z]{2,3}(-[A-Z]{2})?(,[a-z]{2,3}(-[A-Z]{2})?)*$`), - "currency.numeric": regexp.MustCompile(`^\d{3}$`), - "currency.decimals": regexp.MustCompile(`^[0-4]$`), + "territory.numeric": regexp.MustCompile(`^\d{3}$`), + "territory.tld": regexp.MustCompile(`^\.[a-z]{2}$`), + "territory.calling-code": regexp.MustCompile(`^\d{1,4}(-\d{3})?$`), + "territory.capital": regexp.MustCompile(`\p{L}`), + "territory.country": regexp.MustCompile(`^[A-Z]{2}$`), + "territory.currency": regexp.MustCompile(`^[A-Z]{3}$`), + "territory.flag": regexp.MustCompile(`^[\x{1F1E6}-\x{1F1FF}]{2}$`), + "territory.languages": regexp.MustCompile(`^[a-z]{2,3}(-[A-Z]{2})?(,[a-z]{2,3}(-[A-Z]{2})?)*$`), + "currency.numeric": regexp.MustCompile(`^\d{3}$`), + "currency.decimals": regexp.MustCompile(`^[0-4]$`), } for i := 0; i < 100; i++ { for p, rx := range re { @@ -716,10 +721,34 @@ func TestShippedTables(t *testing.T) { } count := map[string]bool{} for i := 0; i < 5000; i++ { - count[fake(t, f, "country.alpha2")] = true + count[fake(t, f, "territory.alpha2")] = true } if len(count) < 200 { - t.Fatalf("country draws %d distinct rows in 5000, want the full register", len(count)) + t.Fatalf("territory draws %d distinct rows in 5000, want the full register", len(count)) + } +} + +// TestEveryTerritoryNamesAShippedCountry proves what no parent can: the sovereign a +// territory names is a row of the same table, which a link would make a cycle. +func TestEveryTerritoryNamesAShippedCountry(t *testing.T) { + data, err := os.ReadFile(filepath.Join("data", "misc", "territory.tsv")) + if err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSuffix(string(data), "\n"), "\n") + head := strings.Split(lines[0], "\t") + alpha2, country := slices.Index(head, "alpha2"), slices.Index(head, "country") + if alpha2 < 0 || country < 0 { + t.Fatalf("territory.tsv columns %v, want alpha2 and country", head) + } + keys := map[string]bool{} + for _, line := range lines[1:] { + keys[strings.Split(line, "\t")[alpha2]] = true + } + for i, line := range lines[1:] { + if c := strings.Split(line, "\t")[country]; !keys[c] { + t.Errorf("territory.tsv line %d: country %q is no alpha2 of the table", i+2, c) + } } } -- 2.52.0 From b0db82f2f11c445b00729042c7f2f4788983943d Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 18 Sep 2026 23:02:45 +0200 Subject: [PATCH 02/14] Rename misc.country to misc.territory, name each one's sovereign state, and state the four data rules that settle it --- CHANGELOG.md | 12 +- DATA-LICENSES.md | 2 +- README.md | 69 ++++-- cmd/fejkdata/main.go | 2 +- data-import/territory.py | 25 +- data/misc/territory.json | 2 +- data/misc/territory.tsv | 484 +++++++++++++++++++-------------------- todo.md | 16 +- 8 files changed, 336 insertions(+), 276 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 357feb2..7815d75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ replacement, and each removed path, column or flag. - First release: the CLI, the library and the shipped data set. - Table categories: a category JSON naming a `rows` TSV beside it, with the options `key`, `name`, `weight` and `parent`; a path selects a row by key or name, - `misc.country[SE]`, and descends to a linked table by name; linked tables draw + `misc.territory[SE]`, and descends to a linked table by name; linked tables draw consistently within one render and draw group. `rows` is an option, so no template may carry a field of that name. A `name` without a `key` resolves inside the table's `parent`. Refused at `New`: a `name` without a `key` or a @@ -21,10 +21,12 @@ replacement, and each removed path, column or flag. row of, or two paths pinning different rows of one table. - `New` refuses a root choice of templates sharing one format and one set of string fields, naming the rows TSV to write instead. -- `misc.country`, `misc.currency`, `misc.language`, `misc.httpstatus` and - `misc.mimetype` are tables. `misc.country` is every ISO 3166 country that has a - capital, a currency and a TLD, with the columns `calling-code`, `capital`, - `currency`, `flag`, `languages`, `numeric` and `tld` added; `misc.currency` the +- `misc.territory`, `misc.currency`, `misc.language`, `misc.httpstatus` and + `misc.mimetype` are tables. `misc.territory` is every ISO 3166-1 territory that has + a capital, a currency and a TLD, with the columns `calling-code`, `capital`, + `country`, `currency`, `flag`, `languages`, `numeric` and `tld` added, `country` + naming the sovereign state it belongs to and itself where it is one; + `misc.currency` the current ISO 4217 currencies with a minor unit, with `decimals` and `numeric` added, and its symbols from CLDR; `misc.language` every ISO 639-2 entry carrying a 639-1 code, with a `code3` column holding its 639-2/T code; `misc.httpstatus` each diff --git a/DATA-LICENSES.md b/DATA-LICENSES.md index 2ff85a6..1a43fb0 100644 --- a/DATA-LICENSES.md +++ b/DATA-LICENSES.md @@ -15,7 +15,7 @@ Every shipped dataset, its source, its licence and the attribution it asks for. | `en_US/first-name.tsv` | [SSA](https://www.ssa.gov/oact/babynames/) baby names, births 1930 to 2020, through [hackerb9/ssa-baby-names](https://github.com/hackerb9/ssa-baby-names) | public domain | none required | `data-import/names-us.py` | | `en_US/last-name.tsv` | Census Bureau surnames occurring 100 or more times, 2010 | public domain | none required | `data-import/names-us.py` | | `sv_SE/sex.tsv`, `sv_SE/birth-number.tsv`, `sv_SE/title.tsv`, `en_US/sex.tsv`, `en_US/title.tsv` | curated (Skatteverket's test birth numbers are facts) | โ€” | โ€” | โ€” | -| `misc/country.tsv` | [datasets/country-codes](https://github.com/datasets/country-codes) | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/country.py` | +| `misc/territory.tsv` | [datasets/country-codes](https://github.com/datasets/country-codes) | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/territory.py` | | `misc/currency.tsv` | [datasets/currency-codes](https://github.com/datasets/currency-codes); symbols from [Unicode CLDR](https://github.com/unicode-org/cldr) `en.xml` and `root.xml` | PDDL 1.0; [Unicode License v3](https://www.unicode.org/license.txt) | CLDR: "Copyright ยฉ 1991-2025 Unicode, Inc. Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries." | `data-import/currency.py` | | `misc/httpstatus.tsv` | [IANA HTTP Status Code Registry](https://www.iana.org/assignments/http-status-codes/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpstatus.py` | | `misc/language.tsv` | [datasets/language-codes](https://github.com/datasets/language-codes), the [Library of Congress](https://www.loc.gov/standards/iso639-2/) ISO 639-2 register | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/language.py` | diff --git a/README.md b/README.md index ad915dc..e532d17 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ fejkdata sv_SE.person.last # Eriksson fejkdata --seed 42 sv_SE.address # the same address every run fejkdata -n 3 --separator ', ' sv_SE.word # nรคt, barn, sol fejkdata --list # every path the data offers -fejkdata 'misc.country[SE].capital' # Stockholm โ€” a table's row, selected by key or name -fejkdata 'geo.SE.locality[Lund].street' # Fjelievรคgen โ€” a linked table, drawn inside the row +fejkdata 'misc.territory[SE].capital' # Stockholm โ€” a table's row, selected by key or name +fejkdata 'geo.SE.locality[Lund].street' # Fjelievรคgen โ€” a linked table, drawn inside the row fejkdata --data-path ./mydata sv_SE.word # layer a directory over the shipped data fejkdata --no-shipped-data -d ./mydata --list # only your data fejkdata 'name: {/sv_SE.person.last}' # name: โ€” an inline template @@ -161,15 +161,20 @@ Each locale carries `address`, `color`, `company`, `date`, `email`, `first-name` `ip`, `last-name`, `person`, `phone`, `price`, `sentence`, `sex`, `time`, `url`, `username`, `version` and `word`, formatted per locale; `sv_SE` adds `personnummer` and `samordningsnummer`, `en_US` adds `ssn` and `itin`. `misc` -carries `car`, `coordinate`, `country` (ISO 3166), `creditcard` (Luhn-valid), -`currency` (ISO 4217), `datetime` (RFC 3339), `emoji`, `httpstatus`, `language` -(ISO 639-1, with its 639-2/T code), `mac`, `mimetype`, `objectid`, `timezone` +carries `car`, `coordinate`, `creditcard` (Luhn-valid), `currency` (ISO 4217), +`datetime` (RFC 3339), `emoji`, `httpstatus`, `language` (ISO 639-1, with its +639-2/T code), `mac`, `mimetype`, `objectid`, `territory` (ISO 3166-1), `timezone` (IANA), `useragent` and `uuid` (v4). Many carry sub-fields โ€” `misc.currency.symbol`, -`misc.country.alpha2`, `misc.httpstatus.code` โ€” which `--list` shows. `country`, -`currency`, `httpstatus`, `language` and `mimetype` are [tables](#table), so -`misc.country[SE].capital` and `misc.currency[Euro].symbol` select a row; +`misc.territory.alpha2`, `misc.httpstatus.code` โ€” which `--list` shows. `currency`, +`httpstatus`, `language`, `mimetype` and `territory` are [tables](#table), so +`misc.territory[SE].capital` and `misc.currency[Euro].symbol` select a row; [`DATA-LICENSES.md`](DATA-LICENSES.md) names each table's source and licence. +ISO 3166-1 codes territories, not sovereign states, so that is what the table is +called: Greenland and ร…land have codes of their own, and `misc.territory.country` +names the state each belongs to โ€” `DK` for Greenland, `FI` for ร…land, and its own +code for a sovereign one. + `sex`, `first-name` and `last-name` are tables weighted by bearers, from SCB, the SSA and the Census Bureau. `first-name` links to `sex`, so `sv_SE.sex[f].first-name` draws a woman's name, and a name both sexes carry is a row under each, so @@ -447,11 +452,11 @@ inline template has no file beside it, so there it stays a choice. ### Row selection -`[key]` or `[name]` after a table's name selects one row: `misc.country[SE]` and -`misc.country[Sweden]` name one row, and `misc.country[SE].capital` reads its column. +`[key]` or `[name]` after a table's name selects one row: `misc.territory[SE]` and +`misc.territory[Sweden]` name one row, and `misc.territory[SE].capital` reads its column. A name naming several rows is an error listing their keys, unless a row selected before it settles which ([Linked tables](#linked-tables)). A selector is part of the path, so it works -wherever a path does: `Fake`, `FakeRecord`, a `{/misc.country[SE].capital}` reference +wherever a path does: `Fake`, `FakeRecord`, a `{/misc.territory[SE].capital}` reference and a struct tag. A dot inside the brackets belongs to the key or name, so `city[St. Louis]` selects it. A path starts with a name, and `[` still opens a JSON array at the start of a CLI argument, so `'[SE]'` alone names nothing. @@ -772,6 +777,13 @@ App developers writing tests and fixtures, in Go and at a shell: 9. **Fast enough to be free** โ€” a value renders in about a microsecond and `New` parses and validates the whole set once upfront, so generating fixtures stays noise against a test's own runtime. +10. **Data is sourced, or on its way there** โ€” a shipped fact, a name, place, + code, id or classification, is read from a register or open dataset by a + [`data-import/`](data-import) script wherever one exists to read; where none + does yet a small hand-written set ships and [`todo.md`](todo.md) carries the + step that replaces it. Only non-factual copy stays authored. A sourced table + holds the rows its source holds: none is added by hand, and one is dropped + only by a rule the script states. ## Decisions @@ -1124,11 +1136,40 @@ App developers writing tests and fixtures, in Go and at a shell: tells the two apart, `sex[f].first-name[Kim]`, the ambiguity error spells each row inside its parent, and a name repeating inside one parent row is refused at load, since nothing could then select it. -- **`misc.country` carries a currency code, it does not link to `misc.currency`.** +- **`misc` is what every locale shares.** A category whose facts differ by country + belongs in that country's locale, read from the register that country's own + records use; `misc` takes only sources that are international. So NHTSA vPIC + builds `en_US.car` and Mobility Sweden's registrations `sv_SE.car`, never + `misc.car`. +- **A register's canonical spelling loses to the one its domain writes.** Where a + source offers several spellings of one fact, the shipped one is what records in + that domain carry. `misc.timezone` reads `zone.tab` and not the `zone1970.tab` + that supersedes it, because the latter keeps one zone per set of countries that + have agreed since 1970: it spells Sweden `Europe/Berlin`, and no Swedish system + writes that. For the same reason `misc.language` takes the ISO 639-2 register's + first synonym over CLDR, which says "Chinese, Mandarin" for `zh`. +- **`misc.territory` is the spine, and a `misc` table naming a territory links to + it.** Where every territory row has a child the column is a `parent`, and the + import drops the child rows whose territory the set does not ship โ€” 17 of + `misc.timezone`'s, Antarctica's ten among them. Agreement across a record is + worth more than the last rows of a table. Where no such link can hold the fact + stays a column. +- **`misc.territory` names its sovereign in a column, and there is no `misc.country` + table.** ISO 3166-1 codes territories, so `territory` is the honest name, and + `is_independent` in the register gives each one's state. A second table of the 195 + sovereigns would hold a `DK` row beside the territory `DK` row, both carrying + Denmark's capital, currency, flag and TLD โ€” two owners for one fact, drifting at + the next import. Splitting the columns to avoid that is worse: put `capital` on + the territory alone and a country row can no longer name Copenhagen. The column + cannot be a `parent`, since a table never reaches its own family; a test proves + every value names a row instead. A territory the register records no state for + stands alone, which is `EH` alone, and naming one for it would be a claim + fejkdata has no business making. +- **`misc.territory` carries a currency code, it does not link to `misc.currency`.** A `parent` demands a child for every parent row, and ISO 4217 registers codes no country's row can name: the funds codes (Mvdol, WIR Euro, US Dollar (Next day)), and VED beside VES, both Venezuela's, of which a country row names one. Linking - would trade the register for the link, and `misc.country.currency` already pairs a + would trade the register for the link, and `misc.territory.currency` already pairs a country with its currency in one draw. - **An extension may name two media types.** `.xml`, `.rtf`, `.sub`, `.mpp` and `.ac` each name two rows of `misc.mimetype`. Separating them would mean dropping a @@ -1207,7 +1248,7 @@ and `names-us.py` rejects a client for a while after a burst, so `--surnames` ta a copy of the surname file: ```sh -docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/country.py +docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/territory.py docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/currency.py docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/geo-us.py docker compose run --rm --user "$(id -u):$(id -g)" -e TRAFIKVERKET_API_KEY data-import data-import/geo-se.py diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index 57ccd79..327e5c3 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -24,7 +24,7 @@ import ( const usage = `Usage: fejkdata [flags] a category, or a dotted path into one (person, person.last); - a table's row by key or name: 'misc.country[SE]', 'misc.country[Sweden].tld' + a table's row by key or name: 'misc.territory[SE]', 'misc.territory[Sweden].tld'