Answer architecture round 3: fence's docstring says a locality needs entries in both child maps, and zcta_localities says what it maps to

data-import/geo-se.py

@@ -186,9 +186,9 @@ def main():
     regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in region_names.items()}
     municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipality_names.items()}
     kept, dropped = geo.fence(regions, municipalities, places, named, {k: l["codes"] for k, l in places.items()})
-    geo.log_dropped(dropped["locality"], "no street or postal code")
+    geo.log_dropped_localities(dropped["locality"])
-    if dropped["municipality"]:
+    if dropped["municipality"] or dropped["region"]:
-        sys.exit(f"municipalities without a locality: {sorted(dropped['municipality'])}")
+        sys.exit(f"left without a child: municipalities {sorted(dropped['municipality'])}, regions {sorted(dropped['region'])}")
     places = kept["locality"]
 
     tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])

data-import/geo-us.py

@@ -126,7 +126,7 @@ def localities(cache, min_population, counties):
 
 
 def zcta_localities(cache, localities):
-    """Each ZCTA whose largest part inside an incorporated place lies in a shipped place."""
+    """Each ZCTA whose largest part inside an incorporated place lies in a shipped place, mapped to that place."""
     parts = {}
     for r in csv.DictReader(io.StringIO(text(source.fetch(ZCTA_PLACE, cache, "zcta-place.txt"))), delimiter="|"):
         if r["GEOID_ZCTA5_20"] and r["GEOID_PLACE_20"] and not r["NAMELSAD_PLACE_20"].endswith(" CDP"):
@@ -176,7 +176,7 @@ def main():
         if addresses[zcta]:
             zctas[locality].append(zcta)
     kept, dropped = geo.fence(regions, counties, places, named, zctas)
-    geo.log_dropped(dropped["locality"], "no street or postal code")
+    geo.log_dropped_localities(dropped["locality"])
     print(f"counties without a place, dropped: {len(dropped['municipality'])}; states: {sorted(dropped['region'])}", file=sys.stderr)
     places = kept["locality"]
 

data-import/geo.py

@@ -73,12 +73,12 @@ def top_streets(count, per_locality, column):
 
 
 def fence(regions, municipalities, localities, streets, postal_codes):
-    """Meet the loader's fence that every parent row has a child row: keep each locality both streets and postal_codes
+    """Meet the loader's fence that every parent row has a child row: keep each locality that has entries in both
-    hold a key for, then each municipality and region left with a child. regions and municipalities map a code to a
+    streets and postal_codes, each a map from a locality's key to its entries, then each municipality and region left
-    row, and a municipality row names its region under "region". localities map a key to a locality row: name,
+    with a child. regions and municipalities map a code to a row, and a municipality row names its region under
-    municipality, population, lat and lon, besides the columns its country adds. streets and postal_codes map a locality's
+    "region". localities map a key to a locality row: name, municipality, population, lat and lon, besides the columns
-    key to its rows. Returns the kept rows, then the dropped rows, each a dict keyed by table: "locality", "municipality"
+    its country adds. Returns the kept rows, then the dropped rows, each a dict keyed by table: "locality",
-    and "region"."""
+    "municipality" and "region"."""
     kept = {"locality": {k: l for k, l in localities.items() if streets.get(k) and postal_codes.get(k)}}
     municipality_codes = {l["municipality"] for l in kept["locality"].values()}
     kept["municipality"] = {c: m for c, m in municipalities.items() if c in municipality_codes}
@@ -89,8 +89,8 @@ def fence(regions, municipalities, localities, streets, postal_codes):
     return kept, dropped
 
 
-def log_dropped(rows, why):
+def log_dropped_localities(rows):
-    """Logs each row as dropped for why, by its key and its name where the two differ."""
+    """Logs each locality row as dropped, by its key and its name where the two differ."""
     for key, row in rows.items():
         label = key if key == row["name"] else f"{key} {row['name']}"
-        print(f"{label}: {why}, dropped", file=sys.stderr)
+        print(f"{label}: no street or postal code, dropped", file=sys.stderr)