Answer architecture round 2: street.tsv is written from the kept localities, fence checks a child holds rows, and its names say what they hold

data-import/geo-se.py

@@ -195,7 +195,8 @@ def main():
     tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(kept["municipality"].items())])
     tsv.write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
     tsv.write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in places.values() for c in l["codes"]), key=lambda r: r["code"]))
-    tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(named) for s in named[locality]])
+    tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(places) for s in named[locality]])
+
 
 if __name__ == "__main__":
     main()

data-import/geo-us.py

@@ -125,7 +125,7 @@ def localities(cache, min_population, counties):
     return out
 
 
-def postal_codes(cache, localities):
+def zcta_localities(cache, localities):
     """Each ZCTA whose largest part inside an incorporated place lies in a shipped place."""
     parts = {}
     for r in csv.DictReader(io.StringIO(text(source.fetch(ZCTA_PLACE, cache, "zcta-place.txt"))), delimiter="|"):
@@ -169,7 +169,7 @@ def main():
     print(f"counties without a population estimate, dropped: {dict(unestimated)}", file=sys.stderr)
 
     places = localities(cache, a.min_population, counties)
-    locality_of_zcta = postal_codes(cache, places)
+    locality_of_zcta = zcta_localities(cache, places)
     named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.streets_per_locality)
     zctas = collections.defaultdict(list)
     for zcta, locality in sorted(locality_of_zcta.items()):
@@ -184,7 +184,8 @@ def main():
     tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(kept["municipality"].items())])
     tsv.write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
     tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], sorted(({"code": z, "locality": l, "addresses": addresses[z]} for l in places for z in zctas[l]), key=lambda r: r["code"]))
-    tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(named) for s in named[locality]])
+    tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(places) for s in named[locality]])
+
 
 if __name__ == "__main__":
     main()

data-import/geo.py

@@ -76,13 +76,14 @@ def fence(regions, municipalities, localities, streets, postal_codes):
     """Meet the loader's fence that every parent row has a child row: keep each locality both streets and postal_codes
     hold a key for, then each municipality and region left with a child. regions and municipalities map a code to a
     row, and a municipality row names its region under "region". localities map a key to a locality row: name,
-    municipality, population, lat and lon, besides the columns its country adds. Returns the kept rows, then the dropped
+    municipality, population, lat and lon, besides the columns its country adds. streets and postal_codes map a locality's
-    rows, each a dict of the three tables by name."""
+    key to its rows. Returns the kept rows, then the dropped rows, each a dict keyed by table: "locality", "municipality"
-    kept = {"locality": {k: l for k, l in localities.items() if k in streets and k in postal_codes}}
+    and "region"."""
-    parents = {l["municipality"] for l in kept["locality"].values()}
+    kept = {"locality": {k: l for k, l in localities.items() if streets.get(k) and postal_codes.get(k)}}
-    kept["municipality"] = {c: m for c, m in municipalities.items() if c in parents}
+    municipality_codes = {l["municipality"] for l in kept["locality"].values()}
-    parents = {m["region"] for m in kept["municipality"].values()}
+    kept["municipality"] = {c: m for c, m in municipalities.items() if c in municipality_codes}
-    kept["region"] = {c: r for c, r in regions.items() if c in parents}
+    region_codes = {m["region"] for m in kept["municipality"].values()}
+    kept["region"] = {c: r for c, r in regions.items() if c in region_codes}
     every = {"locality": localities, "municipality": municipalities, "region": regions}
     dropped = {table: {k: row for k, row in rows.items() if k not in kept[table]} for table, rows in every.items()}
     return kept, dropped