From 32e01565b4d840e5c45b22161b288294c923cc7e Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 9 Oct 2026 12:26:00 +0200 Subject: [PATCH] Answer architecture round 3: fence's docstring says a locality needs entries in both child maps, and zcta_localities says what it maps to --- data-import/geo-se.py | 6 +++--- data-import/geo-us.py | 4 ++-- data-import/geo.py | 18 +++++++++--------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/data-import/geo-se.py b/data-import/geo-se.py index 6a31b05..e0ba179 100644 --- a/data-import/geo-se.py +++ b/data-import/geo-se.py @@ -186,9 +186,9 @@ def main(): regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in region_names.items()} municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipality_names.items()} kept, dropped = geo.fence(regions, municipalities, places, named, {k: l["codes"] for k, l in places.items()}) - geo.log_dropped(dropped["locality"], "no street or postal code") - if dropped["municipality"]: - sys.exit(f"municipalities without a locality: {sorted(dropped['municipality'])}") + geo.log_dropped_localities(dropped["locality"]) + if dropped["municipality"] or dropped["region"]: + sys.exit(f"left without a child: municipalities {sorted(dropped['municipality'])}, regions {sorted(dropped['region'])}") places = kept["locality"] tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())]) diff --git a/data-import/geo-us.py b/data-import/geo-us.py index 187c187..f748e86 100644 --- a/data-import/geo-us.py +++ b/data-import/geo-us.py @@ -126,7 +126,7 @@ def localities(cache, min_population, counties): def zcta_localities(cache, localities): - """Each ZCTA whose largest part inside an incorporated place lies in a shipped place.""" + """Each ZCTA whose largest part inside an incorporated place lies in a shipped place, mapped to that place.""" parts = {} for r in csv.DictReader(io.StringIO(text(source.fetch(ZCTA_PLACE, cache, "zcta-place.txt"))), delimiter="|"): if r["GEOID_ZCTA5_20"] and r["GEOID_PLACE_20"] and not r["NAMELSAD_PLACE_20"].endswith(" CDP"): @@ -176,7 +176,7 @@ def main(): if addresses[zcta]: zctas[locality].append(zcta) kept, dropped = geo.fence(regions, counties, places, named, zctas) - geo.log_dropped(dropped["locality"], "no street or postal code") + geo.log_dropped_localities(dropped["locality"]) print(f"counties without a place, dropped: {len(dropped['municipality'])}; states: {sorted(dropped['region'])}", file=sys.stderr) places = kept["locality"] diff --git a/data-import/geo.py b/data-import/geo.py index a993556..d06431e 100644 --- a/data-import/geo.py +++ b/data-import/geo.py @@ -73,12 +73,12 @@ def top_streets(count, per_locality, column): def fence(regions, municipalities, localities, streets, postal_codes): - """Meet the loader's fence that every parent row has a child row: keep each locality both streets and postal_codes - hold a key for, then each municipality and region left with a child. regions and municipalities map a code to a - row, and a municipality row names its region under "region". localities map a key to a locality row: name, - municipality, population, lat and lon, besides the columns its country adds. streets and postal_codes map a locality's - key to its rows. Returns the kept rows, then the dropped rows, each a dict keyed by table: "locality", "municipality" - and "region".""" + """Meet the loader's fence that every parent row has a child row: keep each locality that has entries in both + streets and postal_codes, each a map from a locality's key to its entries, then each municipality and region left + with a child. regions and municipalities map a code to a row, and a municipality row names its region under + "region". localities map a key to a locality row: name, municipality, population, lat and lon, besides the columns + its country adds. Returns the kept rows, then the dropped rows, each a dict keyed by table: "locality", + "municipality" and "region".""" kept = {"locality": {k: l for k, l in localities.items() if streets.get(k) and postal_codes.get(k)}} municipality_codes = {l["municipality"] for l in kept["locality"].values()} kept["municipality"] = {c: m for c, m in municipalities.items() if c in municipality_codes} @@ -89,8 +89,8 @@ def fence(regions, municipalities, localities, streets, postal_codes): return kept, dropped -def log_dropped(rows, why): - """Logs each row as dropped for why, by its key and its name where the two differ.""" +def log_dropped_localities(rows): + """Logs each locality row as dropped, by its key and its name where the two differ.""" for key, row in rows.items(): label = key if key == row["name"] else f"{key} {row['name']}" - print(f"{label}: {why}, dropped", file=sys.stderr) + print(f"{label}: no street or postal code, dropped", file=sys.stderr)