From c032cb779c2c5837d231795823ebd411ce1006fa Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 9 Oct 2026 12:21:14 +0200 Subject: [PATCH] Answer architecture round 2: street.tsv is written from the kept localities, fence checks a child holds rows, and its names say what they hold --- data-import/geo-se.py | 3 ++- data-import/geo-us.py | 7 ++++--- data-import/geo.py | 15 ++++++++------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/data-import/geo-se.py b/data-import/geo-se.py index aec026f..6a31b05 100644 --- a/data-import/geo-se.py +++ b/data-import/geo-se.py @@ -195,7 +195,8 @@ def main(): tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(kept["municipality"].items())]) tsv.write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())]) tsv.write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in places.values() for c in l["codes"]), key=lambda r: r["code"])) - tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(named) for s in named[locality]]) + tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(places) for s in named[locality]]) + if __name__ == "__main__": main() diff --git a/data-import/geo-us.py b/data-import/geo-us.py index 5df364b..187c187 100644 --- a/data-import/geo-us.py +++ b/data-import/geo-us.py @@ -125,7 +125,7 @@ def localities(cache, min_population, counties): return out -def postal_codes(cache, localities): +def zcta_localities(cache, localities): """Each ZCTA whose largest part inside an incorporated place lies in a shipped place.""" parts = {} for r in csv.DictReader(io.StringIO(text(source.fetch(ZCTA_PLACE, cache, "zcta-place.txt"))), delimiter="|"): @@ -169,7 +169,7 @@ def main(): print(f"counties without a population estimate, dropped: {dict(unestimated)}", file=sys.stderr) places = localities(cache, a.min_population, counties) - locality_of_zcta = postal_codes(cache, places) + locality_of_zcta = zcta_localities(cache, places) named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.streets_per_locality) zctas = collections.defaultdict(list) for zcta, locality in sorted(locality_of_zcta.items()): @@ -184,7 +184,8 @@ def main(): tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(kept["municipality"].items())]) tsv.write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())]) tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], sorted(({"code": z, "locality": l, "addresses": addresses[z]} for l in places for z in zctas[l]), key=lambda r: r["code"])) - tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(named) for s in named[locality]]) + tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(places) for s in named[locality]]) + if __name__ == "__main__": main() diff --git a/data-import/geo.py b/data-import/geo.py index c374c4b..a993556 100644 --- a/data-import/geo.py +++ b/data-import/geo.py @@ -76,13 +76,14 @@ def fence(regions, municipalities, localities, streets, postal_codes): """Meet the loader's fence that every parent row has a child row: keep each locality both streets and postal_codes hold a key for, then each municipality and region left with a child. regions and municipalities map a code to a row, and a municipality row names its region under "region". localities map a key to a locality row: name, - municipality, population, lat and lon, besides the columns its country adds. Returns the kept rows, then the dropped - rows, each a dict of the three tables by name.""" - kept = {"locality": {k: l for k, l in localities.items() if k in streets and k in postal_codes}} - parents = {l["municipality"] for l in kept["locality"].values()} - kept["municipality"] = {c: m for c, m in municipalities.items() if c in parents} - parents = {m["region"] for m in kept["municipality"].values()} - kept["region"] = {c: r for c, r in regions.items() if c in parents} + municipality, population, lat and lon, besides the columns its country adds. streets and postal_codes map a locality's + key to its rows. Returns the kept rows, then the dropped rows, each a dict keyed by table: "locality", "municipality" + and "region".""" + kept = {"locality": {k: l for k, l in localities.items() if streets.get(k) and postal_codes.get(k)}} + municipality_codes = {l["municipality"] for l in kept["locality"].values()} + kept["municipality"] = {c: m for c, m in municipalities.items() if c in municipality_codes} + region_codes = {m["region"] for m in kept["municipality"].values()} + kept["region"] = {c: r for c, r in regions.items() if c in region_codes} every = {"locality": localities, "municipality": municipalities, "region": regions} dropped = {table: {k: row for k, row in rows.items() if k not in kept[table]} for table, rows in every.items()} return kept, dropped