data-import/geo-se.py

@@ -178,21 +178,24 @@ def main():
         sys.exit("set TRAFIKVERKET_API_KEY or pass --key-file")
     cache, out = geo.directories(a)
 
-    regions, municipalities = scb_codes(cache)
+    region_names, municipality_names = scb_codes(cache)
     population = scb_population(cache)
     codes = geonames(cache)
-    places = localities(codes, scb_tatorter(cache), municipalities, population)
+    places = localities(codes, scb_tatorter(cache), municipality_names, population)
     named = streets(nvdb_segments(cache, key), codes, places, a.streets_per_locality)
-    regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in regions.items()}
-    municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipalities.items()}
-    regions, municipalities, places = geo.fence(regions, municipalities, places, named)
+    regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in region_names.items()}
+    municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipality_names.items()}
+    kept, dropped = geo.fence(regions, municipalities, places, named, {k: l["codes"] for k, l in places.items()})
+    geo.log_dropped(dropped["locality"], "no street or postal code")
+    if dropped["municipality"]:
+        sys.exit(f"municipalities without a locality: {sorted(dropped['municipality'])}")
+    places = kept["locality"]
 
-    tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(regions.items())])
-    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(municipalities.items())])
+    tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
+    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(kept["municipality"].items())])
     tsv.write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
     tsv.write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in places.values() for c in l["codes"]), key=lambda r: r["code"]))
     tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(named) for s in named[locality]])
 
-
 if __name__ == "__main__":
     main()

data-import/geo-us.py

@@ -171,14 +171,20 @@ def main():
     places = localities(cache, a.min_population, counties)
     locality_of_zcta = postal_codes(cache, places)
     named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.streets_per_locality)
-    regions, counties, places = geo.fence(regions, counties, places, named)
+    zctas = collections.defaultdict(list)
+    for zcta, locality in sorted(locality_of_zcta.items()):
+        if addresses[zcta]:
+            zctas[locality].append(zcta)
+    kept, dropped = geo.fence(regions, counties, places, named, zctas)
+    geo.log_dropped(dropped["locality"], "no street or postal code")
+    print(f"counties without a place, dropped: {len(dropped['municipality'])}; states: {sorted(dropped['region'])}", file=sys.stderr)
+    places = kept["locality"]
 
-    tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(regions.items())])
-    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(counties.items())])
+    tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
+    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(kept["municipality"].items())])
     tsv.write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
-    tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], [{"code": z, "locality": l, "addresses": addresses[z]} for z, l in sorted(locality_of_zcta.items()) if addresses[z] and l in places])
+    tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], sorted(({"code": z, "locality": l, "addresses": addresses[z]} for l in places for z in zctas[l]), key=lambda r: r["code"]))
     tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(named) for s in named[locality]])
 
-
 if __name__ == "__main__":
     main()

data-import/geo.py

@@ -72,22 +72,24 @@ def top_streets(count, per_locality, column):
     return {locality: [{"name": name, "locality": locality, column: n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:per_locality]] for locality, named in of.items()}
 
 
-def fence(regions, municipalities, localities, streets):
-    """Meet the loader's fence that every parent row has a child row: drop each locality streets holds no rows for, then
-    each municipality and region left without one, logging each drop. regions and municipalities map a code to a row,
-    and a municipality row names its region under "region". localities map a key to a locality row: name, municipality,
-    population, lat and lon, besides the columns its country adds. streets maps a locality's key to its street rows.
-    Returns the kept regions, municipalities and localities."""
-    localities = kept(localities, streets, "streets")
-    municipalities = kept(municipalities, {l["municipality"] for l in localities.values()}, "localities")
-    regions = kept(regions, {m["region"] for m in municipalities.values()}, "municipalities")
-    return regions, municipalities, localities
+def fence(regions, municipalities, localities, streets, postal_codes):
+    """Meet the loader's fence that every parent row has a child row: keep each locality both streets and postal_codes
+    hold a key for, then each municipality and region left with a child. regions and municipalities map a code to a
+    row, and a municipality row names its region under "region". localities map a key to a locality row: name,
+    municipality, population, lat and lon, besides the columns its country adds. Returns the kept rows, then the dropped
+    rows, each a dict of the three tables by name."""
+    kept = {"locality": {k: l for k, l in localities.items() if k in streets and k in postal_codes}}
+    parents = {l["municipality"] for l in kept["locality"].values()}
+    kept["municipality"] = {c: m for c, m in municipalities.items() if c in parents}
+    parents = {m["region"] for m in kept["municipality"].values()}
+    kept["region"] = {c: r for c, r in regions.items() if c in parents}
+    every = {"locality": localities, "municipality": municipalities, "region": regions}
+    dropped = {table: {k: row for k, row in rows.items() if k not in kept[table]} for table, rows in every.items()}
+    return kept, dropped
 
 
-def kept(parents, children, kind):
-    """The parents whose key children holds; logs each other parent as dropped."""
-    for key, parent in parents.items():
-        if key not in children:
-            label = key if key == parent["name"] else f"{key} {parent['name']}"
-            print(f"{label}: no {kind}, dropped", file=sys.stderr)
-    return {key: parent for key, parent in parents.items() if key in children}
+def log_dropped(rows, why):
+    """Logs each row as dropped for why, by its key and its name where the two differ."""
+    for key, row in rows.items():
+        label = key if key == row["name"] else f"{key} {row['name']}"
+        print(f"{label}: {why}, dropped", file=sys.stderr)

todo.md

@@ -6,7 +6,7 @@
 
 `Bar = 9`
 
-`Next ID = 168`
+`Next ID = 169`
 
 | Goal | W |
 |---|---|
@@ -118,6 +118,7 @@
 | 79 | 0.2.0 | decision | **Beside a `--data-path`, load in `New` the user's categories, the shipped ones they override or read, and the shipped ones whose reads reach an overridden one; load the rest on first reach.** | 6 | 6 | 7 | 8 | 13 | 12.9 |
 | 27 | 0.2.0 |  | **Merge `en_US.ip` and `sv_SE.ip`, today byte-identical, into one `misc.ip`.** | 3 | 2 | 4 | 6 | 8 | 12.3 |
 | 107 | 0.2.0 | decision | **Move the template engine into `internal/`, leaving the root the public API and its options.** | 6 | 9 | 3 | 8 | 3.2, 3.4 | 12.2 |
+| 168 | 0.2.0 | principle | **Test the shared steps of `data-import/` in the merge gate, starting with `geo.fence`.** | 2 | 4 | 2 | 6 | 3 | 11.8 |
 | 161 | 0.2.0 |  | **Let a field whose name starts or ends with whitespace be bound to a name, or refuse such an identifier, so `{ x as p}` means one thing.** | 2 | 2 | 2 | 6 | 5.6, 4.1 | 11.7 |
 | 112 | 0.2.0 |  | **Check a calc's field operands where its template resolves, beside its name operands, once item 36 reports every mistake at once.** | 2 | 2 | 1 | 6 | 3.2 | 10.3 |
 | 108 | 0.2.0 | defect | **Name item 26 in the decision "No builtin reads the clock, so a date is bounded by days, never by an age", as the item that ends its conflict with goal 4.** | 1 | 1 | 1 | 4 | 3.4 | 8.0 |
@@ -441,6 +442,10 @@ The decision "With only the shipped set, a category loads on the first call reac
 
 The engine returns column values and the root builds `Record`. Revises the decision "The template engine stays the root package until item 107; what reads no engine type, a table's rows included, sits in `internal/`", and, if the vocabulary moves, "The vocabulary sits below `doc.go`'s package clause, not in the package doc".
 
+### 168. Test the shared steps of `data-import/` in the merge gate, starting with `geo.fence`.
+
+Filed by the principles audit of item 148's chunk, against "Most, if not all, code should have automated tests" in `technical-principles.md`. No Python under `data-import/` has a test, and the gate, `docker build .`, runs Go alone. Item 148 checked `geo.fence` by regenerating both countries' TSVs byte-identical from the source cache, which needs the full downloads. Item 61 puts a geo script per country on `fence`.
+
 ### 161. Let a field whose name starts or ends with whitespace be bound to a name, or refuse such an identifier, so `{ x as p}` means one thing.
 
 `grammar/identifier.go` allows a field named ` x`, and `{ x}` reads it, but `checkBind` in `format.go` refuses every padded binding, so no spelling binds that field to a name (goal 4.1), and the refusal names one meaning only (goal 5.6). The goals audit of the narrowed goal 5.7 found it.