Answer prose round 1: fence's docstring splits rule, shapes and result, each drop message names what was missing, the US geo row says a place needs a street, and file item 169

data-import/geo-se.py

@@ -188,7 +188,7 @@ def main():
     kept, dropped = geo.fence(regions, municipalities, places, named, {k: l["codes"] for k, l in places.items()})
     geo.log_dropped_localities(dropped["locality"])
     if dropped["municipality"] or dropped["region"]:
-        sys.exit(f"left without a child: municipalities {sorted(dropped['municipality'])}, regions {sorted(dropped['region'])}")
+        sys.exit(f"municipalities without a locality: {sorted(dropped['municipality'])}; regions without a municipality: {sorted(dropped['region'])}")
     places = kept["locality"]
 
     tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])

data-import/geo-us.py

@@ -126,7 +126,7 @@ def localities(cache, min_population, counties):
 
 
 def zcta_localities(cache, localities):
-    """Each ZCTA whose largest part inside an incorporated place lies in a shipped place, mapped to that place."""
+    """Each ZCTA mapped to the incorporated place holding most of its land inside incorporated places, where that place is shipped."""
     parts = {}
     for r in csv.DictReader(io.StringIO(text(source.fetch(ZCTA_PLACE, cache, "zcta-place.txt"))), delimiter="|"):
         if r["GEOID_ZCTA5_20"] and r["GEOID_PLACE_20"] and not r["NAMELSAD_PLACE_20"].endswith(" CDP"):
@@ -177,7 +177,7 @@ def main():
             zctas[locality].append(zcta)
     kept, dropped = geo.fence(regions, counties, places, named, zctas)
     geo.log_dropped_localities(dropped["locality"])
-    print(f"counties without a place, dropped: {len(dropped['municipality'])}; states: {sorted(dropped['region'])}", file=sys.stderr)
+    print(f"counties without a place, dropped: {len(dropped['municipality'])}; states without a county, dropped: {sorted(dropped['region'])}", file=sys.stderr)
     places = kept["locality"]
 
     tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])

data-import/geo.py

@@ -73,12 +73,14 @@ def top_streets(count, per_locality, column):
 
 
 def fence(regions, municipalities, localities, streets, postal_codes):
-    """Meet the loader's fence that every parent row has a child row: keep each locality that has entries in both
-    streets and postal_codes, each a map from a locality's key to its entries, then each municipality and region left
-    with a child. regions and municipalities map a code to a row, and a municipality row names its region under
-    "region". localities map a key to a locality row: name, municipality, population, lat and lon, besides the columns
-    its country adds. Returns the kept rows, then the dropped rows, each a dict keyed by table: "locality",
-    "municipality" and "region"."""
+    """Meet the loader's fence that every parent row has a child row. Keeps each locality with entries in both streets
+    and postal_codes, then each municipality and region left with a child.
+
+    regions and municipalities map a code to a row; a municipality row names its region under "region". localities,
+    streets and postal_codes map a locality's key to its row, its streets and its postal codes. A locality row holds
+    name, municipality, population, lat and lon, plus its country's own keys.
+
+    Returns kept, then dropped: each maps "locality", "municipality" and "region" to that table's rows."""
     kept = {"locality": {k: l for k, l in localities.items() if streets.get(k) and postal_codes.get(k)}}
     municipality_codes = {l["municipality"] for l in kept["locality"].values()}
     kept["municipality"] = {c: m for c, m in municipalities.items() if c in municipality_codes}
@@ -90,7 +92,7 @@ def fence(regions, municipalities, localities, streets, postal_codes):
 
 
 def log_dropped_localities(rows):
-    """Logs each locality row as dropped, by its key and its name where the two differ."""
+    """Logs each locality row as dropped, by its key, plus its name where that differs."""
     for key, row in rows.items():
         label = key if key == row["name"] else f"{key} {row['name']}"
-        print(f"{label}: no street or postal code, dropped", file=sys.stderr)
+        print(f"{label}: no street or no postal code, dropped", file=sys.stderr)