diff --git a/data-import/geo-se.py b/data-import/geo-se.py index 0d3d25c..aec026f 100644 --- a/data-import/geo-se.py +++ b/data-import/geo-se.py @@ -178,21 +178,24 @@ def main(): sys.exit("set TRAFIKVERKET_API_KEY or pass --key-file") cache, out = geo.directories(a) - regions, municipalities = scb_codes(cache) + region_names, municipality_names = scb_codes(cache) population = scb_population(cache) codes = geonames(cache) - places = localities(codes, scb_tatorter(cache), municipalities, population) + places = localities(codes, scb_tatorter(cache), municipality_names, population) named = streets(nvdb_segments(cache, key), codes, places, a.streets_per_locality) - regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in regions.items()} - municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipalities.items()} - regions, municipalities, places = geo.fence(regions, municipalities, places, named) + regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in region_names.items()} + municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipality_names.items()} + kept, dropped = geo.fence(regions, municipalities, places, named, {k: l["codes"] for k, l in places.items()}) + geo.log_dropped(dropped["locality"], "no street or postal code") + if dropped["municipality"]: + sys.exit(f"municipalities without a locality: {sorted(dropped['municipality'])}") + places = kept["locality"] - tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(regions.items())]) - tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(municipalities.items())]) + tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())]) + tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(kept["municipality"].items())]) tsv.write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())]) tsv.write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in places.values() for c in l["codes"]), key=lambda r: r["code"])) tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(named) for s in named[locality]]) - if __name__ == "__main__": main() diff --git a/data-import/geo-us.py b/data-import/geo-us.py index b2fa40a..5df364b 100644 --- a/data-import/geo-us.py +++ b/data-import/geo-us.py @@ -171,14 +171,20 @@ def main(): places = localities(cache, a.min_population, counties) locality_of_zcta = postal_codes(cache, places) named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.streets_per_locality) - regions, counties, places = geo.fence(regions, counties, places, named) + zctas = collections.defaultdict(list) + for zcta, locality in sorted(locality_of_zcta.items()): + if addresses[zcta]: + zctas[locality].append(zcta) + kept, dropped = geo.fence(regions, counties, places, named, zctas) + geo.log_dropped(dropped["locality"], "no street or postal code") + print(f"counties without a place, dropped: {len(dropped['municipality'])}; states: {sorted(dropped['region'])}", file=sys.stderr) + places = kept["locality"] - tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(regions.items())]) - tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(counties.items())]) + tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())]) + tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(kept["municipality"].items())]) tsv.write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())]) - tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], [{"code": z, "locality": l, "addresses": addresses[z]} for z, l in sorted(locality_of_zcta.items()) if addresses[z] and l in places]) + tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], sorted(({"code": z, "locality": l, "addresses": addresses[z]} for l in places for z in zctas[l]), key=lambda r: r["code"])) tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(named) for s in named[locality]]) - if __name__ == "__main__": main() diff --git a/data-import/geo.py b/data-import/geo.py index f282933..c374c4b 100644 --- a/data-import/geo.py +++ b/data-import/geo.py @@ -72,22 +72,24 @@ def top_streets(count, per_locality, column): return {locality: [{"name": name, "locality": locality, column: n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:per_locality]] for locality, named in of.items()} -def fence(regions, municipalities, localities, streets): - """Meet the loader's fence that every parent row has a child row: drop each locality streets holds no rows for, then - each municipality and region left without one, logging each drop. regions and municipalities map a code to a row, - and a municipality row names its region under "region". localities map a key to a locality row: name, municipality, - population, lat and lon, besides the columns its country adds. streets maps a locality's key to its street rows. - Returns the kept regions, municipalities and localities.""" - localities = kept(localities, streets, "streets") - municipalities = kept(municipalities, {l["municipality"] for l in localities.values()}, "localities") - regions = kept(regions, {m["region"] for m in municipalities.values()}, "municipalities") - return regions, municipalities, localities +def fence(regions, municipalities, localities, streets, postal_codes): + """Meet the loader's fence that every parent row has a child row: keep each locality both streets and postal_codes + hold a key for, then each municipality and region left with a child. regions and municipalities map a code to a + row, and a municipality row names its region under "region". localities map a key to a locality row: name, + municipality, population, lat and lon, besides the columns its country adds. Returns the kept rows, then the dropped + rows, each a dict of the three tables by name.""" + kept = {"locality": {k: l for k, l in localities.items() if k in streets and k in postal_codes}} + parents = {l["municipality"] for l in kept["locality"].values()} + kept["municipality"] = {c: m for c, m in municipalities.items() if c in parents} + parents = {m["region"] for m in kept["municipality"].values()} + kept["region"] = {c: r for c, r in regions.items() if c in parents} + every = {"locality": localities, "municipality": municipalities, "region": regions} + dropped = {table: {k: row for k, row in rows.items() if k not in kept[table]} for table, rows in every.items()} + return kept, dropped -def kept(parents, children, kind): - """The parents whose key children holds; logs each other parent as dropped.""" - for key, parent in parents.items(): - if key not in children: - label = key if key == parent["name"] else f"{key} {parent['name']}" - print(f"{label}: no {kind}, dropped", file=sys.stderr) - return {key: parent for key, parent in parents.items() if key in children} +def log_dropped(rows, why): + """Logs each row as dropped for why, by its key and its name where the two differ.""" + for key, row in rows.items(): + label = key if key == row["name"] else f"{key} {row['name']}" + print(f"{label}: {why}, dropped", file=sys.stderr) diff --git a/todo.md b/todo.md index e564559..20709db 100644 --- a/todo.md +++ b/todo.md @@ -6,7 +6,7 @@ `Bar = 9` -`Next ID = 168` +`Next ID = 169` | Goal | W | |---|---| @@ -118,6 +118,7 @@ | 79 | 0.2.0 | decision | **Beside a `--data-path`, load in `New` the user's categories, the shipped ones they override or read, and the shipped ones whose reads reach an overridden one; load the rest on first reach.** | 6 | 6 | 7 | 8 | 13 | 12.9 | | 27 | 0.2.0 | | **Merge `en_US.ip` and `sv_SE.ip`, today byte-identical, into one `misc.ip`.** | 3 | 2 | 4 | 6 | 8 | 12.3 | | 107 | 0.2.0 | decision | **Move the template engine into `internal/`, leaving the root the public API and its options.** | 6 | 9 | 3 | 8 | 3.2, 3.4 | 12.2 | +| 168 | 0.2.0 | principle | **Test the shared steps of `data-import/` in the merge gate, starting with `geo.fence`.** | 2 | 4 | 2 | 6 | 3 | 11.8 | | 161 | 0.2.0 | | **Let a field whose name starts or ends with whitespace be bound to a name, or refuse such an identifier, so `{ x as p}` means one thing.** | 2 | 2 | 2 | 6 | 5.6, 4.1 | 11.7 | | 112 | 0.2.0 | | **Check a calc's field operands where its template resolves, beside its name operands, once item 36 reports every mistake at once.** | 2 | 2 | 1 | 6 | 3.2 | 10.3 | | 108 | 0.2.0 | defect | **Name item 26 in the decision "No builtin reads the clock, so a date is bounded by days, never by an age", as the item that ends its conflict with goal 4.** | 1 | 1 | 1 | 4 | 3.4 | 8.0 | @@ -441,6 +442,10 @@ The decision "With only the shipped set, a category loads on the first call reac The engine returns column values and the root builds `Record`. Revises the decision "The template engine stays the root package until item 107; what reads no engine type, a table's rows included, sits in `internal/`", and, if the vocabulary moves, "The vocabulary sits below `doc.go`'s package clause, not in the package doc". +### 168. Test the shared steps of `data-import/` in the merge gate, starting with `geo.fence`. + +Filed by the principles audit of item 148's chunk, against "Most, if not all, code should have automated tests" in `technical-principles.md`. No Python under `data-import/` has a test, and the gate, `docker build .`, runs Go alone. Item 148 checked `geo.fence` by regenerating both countries' TSVs byte-identical from the source cache, which needs the full downloads. Item 61 puts a geo script per country on `fence`. + ### 161. Let a field whose name starts or ends with whitespace be bound to a name, or refuse such an identifier, so `{ x as p}` means one thing. `grammar/identifier.go` allows a field named ` x`, and `{ x}` reads it, but `checkBind` in `format.go` refuses every padded binding, so no spelling binds that field to a name (goal 4.1), and the refusal names one meaning only (goal 5.6). The goals audit of the narrowed goal 5.7 found it.