diff --git a/README.md b/README.md index 5b54df9..d32992e 100644 --- a/README.md +++ b/README.md @@ -285,7 +285,7 @@ neighbouring one. |-------|----------|----------|--------| | `region` | län, by code or name | state, by USPS abbreviation or name; `code` is the FIPS code | population | | `municipality` | kommun, by code or name | county, by FIPS code or name | population | -| `locality` | postort, by name; [`geo-se.py`](data-import/geo-se.py)'s `localities` picks the candidates, and one with no street is dropped | incorporated place of 25,000 people or more with a postal code of its own, by GEOID or name; Hawaii has none | tätort population, the kommun's where the postort names it, else 200; place population | +| `locality` | postort, by name; [`geo-se.py`](data-import/geo-se.py)'s `localities` picks the candidates, and one with no street is dropped | incorporated place of 25,000 people or more with a postal code and a street of its own, by GEOID or name; Hawaii has none | tätort population, the kommun's where the postort names it, else 200; place population | | `postal-code` | postnummer with street delivery, by code | ZCTA, by code | one; address ranges | | `street` | gatunamn, the ten with most road segments per postort | street name, the ten with most address ranges per place | segments; address ranges | diff --git a/data-import/geo-se.py b/data-import/geo-se.py index e0ba179..439b471 100644 --- a/data-import/geo-se.py +++ b/data-import/geo-se.py @@ -188,7 +188,7 @@ def main(): kept, dropped = geo.fence(regions, municipalities, places, named, {k: l["codes"] for k, l in places.items()}) geo.log_dropped_localities(dropped["locality"]) if dropped["municipality"] or dropped["region"]: - sys.exit(f"left without a child: municipalities {sorted(dropped['municipality'])}, regions {sorted(dropped['region'])}") + sys.exit(f"municipalities without a locality: {sorted(dropped['municipality'])}; regions without a municipality: {sorted(dropped['region'])}") places = kept["locality"] tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())]) diff --git a/data-import/geo-us.py b/data-import/geo-us.py index f748e86..f3582a2 100644 --- a/data-import/geo-us.py +++ b/data-import/geo-us.py @@ -126,7 +126,7 @@ def localities(cache, min_population, counties): def zcta_localities(cache, localities): - """Each ZCTA whose largest part inside an incorporated place lies in a shipped place, mapped to that place.""" + """Each ZCTA mapped to the incorporated place holding most of its land inside incorporated places, where that place is shipped.""" parts = {} for r in csv.DictReader(io.StringIO(text(source.fetch(ZCTA_PLACE, cache, "zcta-place.txt"))), delimiter="|"): if r["GEOID_ZCTA5_20"] and r["GEOID_PLACE_20"] and not r["NAMELSAD_PLACE_20"].endswith(" CDP"): @@ -177,7 +177,7 @@ def main(): zctas[locality].append(zcta) kept, dropped = geo.fence(regions, counties, places, named, zctas) geo.log_dropped_localities(dropped["locality"]) - print(f"counties without a place, dropped: {len(dropped['municipality'])}; states: {sorted(dropped['region'])}", file=sys.stderr) + print(f"counties without a place, dropped: {len(dropped['municipality'])}; states without a county, dropped: {sorted(dropped['region'])}", file=sys.stderr) places = kept["locality"] tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())]) diff --git a/data-import/geo.py b/data-import/geo.py index d06431e..8f2cab7 100644 --- a/data-import/geo.py +++ b/data-import/geo.py @@ -73,12 +73,14 @@ def top_streets(count, per_locality, column): def fence(regions, municipalities, localities, streets, postal_codes): - """Meet the loader's fence that every parent row has a child row: keep each locality that has entries in both - streets and postal_codes, each a map from a locality's key to its entries, then each municipality and region left - with a child. regions and municipalities map a code to a row, and a municipality row names its region under - "region". localities map a key to a locality row: name, municipality, population, lat and lon, besides the columns - its country adds. Returns the kept rows, then the dropped rows, each a dict keyed by table: "locality", - "municipality" and "region".""" + """Meet the loader's fence that every parent row has a child row. Keeps each locality with entries in both streets + and postal_codes, then each municipality and region left with a child. + + regions and municipalities map a code to a row; a municipality row names its region under "region". localities, + streets and postal_codes map a locality's key to its row, its streets and its postal codes. A locality row holds + name, municipality, population, lat and lon, plus its country's own keys. + + Returns kept, then dropped: each maps "locality", "municipality" and "region" to that table's rows.""" kept = {"locality": {k: l for k, l in localities.items() if streets.get(k) and postal_codes.get(k)}} municipality_codes = {l["municipality"] for l in kept["locality"].values()} kept["municipality"] = {c: m for c, m in municipalities.items() if c in municipality_codes} @@ -90,7 +92,7 @@ def fence(regions, municipalities, localities, streets, postal_codes): def log_dropped_localities(rows): - """Logs each locality row as dropped, by its key and its name where the two differ.""" + """Logs each locality row as dropped, by its key, plus its name where that differs.""" for key, row in rows.items(): label = key if key == row["name"] else f"{key} {row['name']}" - print(f"{label}: no street or postal code, dropped", file=sys.stderr) + print(f"{label}: no street or no postal code, dropped", file=sys.stderr) diff --git a/todo.md b/todo.md index 20709db..c72c90f 100644 --- a/todo.md +++ b/todo.md @@ -6,7 +6,7 @@ `Bar = 9` -`Next ID = 169` +`Next ID = 170` | Goal | W | |---|---| @@ -122,6 +122,7 @@ | 161 | 0.2.0 | | **Let a field whose name starts or ends with whitespace be bound to a name, or refuse such an identifier, so `{ x as p}` means one thing.** | 2 | 2 | 2 | 6 | 5.6, 4.1 | 11.7 | | 112 | 0.2.0 | | **Check a calc's field operands where its template resolves, beside its name operands, once item 36 reports every mistake at once.** | 2 | 2 | 1 | 6 | 3.2 | 10.3 | | 108 | 0.2.0 | defect | **Name item 26 in the decision "No builtin reads the clock, so a date is bounded by days, never by an age", as the item that ends its conflict with goal 4.** | 1 | 1 | 1 | 4 | 3.4 | 8.0 | +| 169 | 0.2.0 | question | **Reword the decision "A parent row with no child row is a load error" to say what each geo import does with such a row: drop it, or exit where `geo-se.py` would lose a kommun or a län.** | 1 | 1 | 1 | 4 | 3 | 8.0 | | 38 | 0.3.0 | defect | **Compare `calcParser.binary`'s operator as a rune.** | 1 | 1 | 3 | 6 | 6 | 13.8 | | 45 | 0.3.0 | | **Name the node a selector follows in `childNamed`'s refusal.** | 1 | 1 | 3 | 6 | 6 | 13.8 | | 49 | 0.3.0 | | **Report a CRLF rows file holding only its header as having no rows.** | 1 | 1 | 3 | 6 | 6 | 13.8 | @@ -444,7 +445,7 @@ The engine returns column values and the root builds `Record`. Revises the decis ### 168. Test the shared steps of `data-import/` in the merge gate, starting with `geo.fence`. -Filed by the principles audit of item 148's chunk, against "Most, if not all, code should have automated tests" in `technical-principles.md`. No Python under `data-import/` has a test, and the gate, `docker build .`, runs Go alone. Item 148 checked `geo.fence` by regenerating both countries' TSVs byte-identical from the source cache, which needs the full downloads. Item 61 puts a geo script per country on `fence`. +Against "Most, if not all, code should have automated tests" in `technical-principles.md`: no Python under `data-import/` has a test, and the gate, `docker build .`, runs Go alone. Today `geo.fence` is checked only by regenerating both countries' TSVs from the full downloads and diffing them. Item 61 adds a geo script per country, each calling `fence`. ### 161. Let a field whose name starts or ends with whitespace be bound to a name, or refuse such an identifier, so `{ x as p}` means one thing. @@ -458,6 +459,10 @@ Needs item 36. `checkCalcFields` runs at compile and `checkCalcNames` in `resolv Item 26 revises the decision, but the decision does not point back to it. +### 169. Reword the decision "A parent row with no child row is a load error" to say what each geo import does with such a row: drop it, or exit where `geo-se.py` would lose a kommun or a län. + +Its entry says "The import script drops or fills such rows". There are two geo imports, and `geo-se.py` exits when `geo.fence` drops a kommun or a län, while `geo-us.py` drops counties and states. Rewording a decision is the maintainer's call. + ### 38. Compare `calcParser.binary`'s operator as a rune. `byte(p.rs[p.pos])` reads U+012B `ī` as `+`, so `{calc(a ī b)}` compiles as `a + b`.