Meet the parent-has-a-child fence in one shared geo.fence step, which states the locality row each country hands it

data-import/geo-se.py

@@ -183,13 +183,12 @@ def main():
     codes = geonames(cache)
     places = localities(codes, scb_tatorter(cache), municipalities, population)
     named = streets(nvdb_segments(cache, key), codes, places, a.streets_per_locality)
-    places = geo.with_streets(places, named)
+    regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in regions.items()}
-    empty = sorted(m for m in municipalities if not any(l["municipality"] == m for l in places.values()))
+    municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipalities.items()}
-    if empty:
+    regions, municipalities, places = geo.fence(regions, municipalities, places, named)
-        sys.exit(f"municipalities without a locality: {empty}")
 
-    tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [{"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in sorted(regions.items())])
+    tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(regions.items())])
-    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [{"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in sorted(municipalities.items())])
+    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(municipalities.items())])
     tsv.write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
     tsv.write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in places.values() for c in l["codes"]), key=lambda r: r["code"]))
     tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(named) for s in named[locality]])

data-import/geo-us.py

@@ -171,12 +171,10 @@ def main():
     places = localities(cache, a.min_population, counties)
     locality_of_zcta = postal_codes(cache, places)
     named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.streets_per_locality)
-    places = geo.with_streets(places, named)
+    regions, counties, places = geo.fence(regions, counties, places, named)
-    kept_counties = {l["municipality"] for l in places.values()}
-    kept_regions = {counties[c]["region"] for c in kept_counties}
 
-    tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(regions.items()) if r["abbr"] in kept_regions])
+    tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(regions.items())])
-    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(counties.items()) if c["code"] in kept_counties])
+    tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(counties.items())])
     tsv.write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
     tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], [{"code": z, "locality": l, "addresses": addresses[z]} for z, l in sorted(locality_of_zcta.items()) if addresses[z] and l in places])
     tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(named) for s in named[locality]])

data-import/geo.py

@@ -72,10 +72,22 @@ def top_streets(count, per_locality, column):
     return {locality: [{"name": name, "locality": locality, column: n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:per_locality]] for locality, named in of.items()}
 
 
-def with_streets(places, named):
+def fence(regions, municipalities, localities, streets):
-    """The places that named holds streets for; logs each other place as dropped."""
+    """Meet the loader's fence that every parent row has a child row: drop each locality streets holds no rows for, then
-    for key, place in places.items():
+    each municipality and region left without one, logging each drop. regions and municipalities map a code to a row,
-        if key not in named:
+    and a municipality row names its region under "region". localities map a key to a locality row: name, municipality,
-            label = key if key == place["name"] else f"{key} {place['name']}"
+    population, lat and lon, besides the columns its country adds. streets maps a locality's key to its street rows.
-            print(f"{label}: no streets, dropped", file=sys.stderr)
+    Returns the kept regions, municipalities and localities."""
-    return {key: place for key, place in places.items() if key in named}
+    localities = kept(localities, streets, "streets")
+    municipalities = kept(municipalities, {l["municipality"] for l in localities.values()}, "localities")
+    regions = kept(regions, {m["region"] for m in municipalities.values()}, "municipalities")
+    return regions, municipalities, localities
+
+
+def kept(parents, children, kind):
+    """The parents whose key children holds; logs each other parent as dropped."""
+    for key, parent in parents.items():
+        if key not in children:
+            label = key if key == parent["name"] else f"{key} {parent['name']}"
+            print(f"{label}: no {kind}, dropped", file=sys.stderr)
+    return {key: parent for key, parent in parents.items() if key in children}