diff --git a/data-import/geo-se.py b/data-import/geo-se.py index c5f1b57..0d3d25c 100644 --- a/data-import/geo-se.py +++ b/data-import/geo-se.py @@ -183,13 +183,12 @@ def main(): codes = geonames(cache) places = localities(codes, scb_tatorter(cache), municipalities, population) named = streets(nvdb_segments(cache, key), codes, places, a.streets_per_locality) - places = geo.with_streets(places, named) - empty = sorted(m for m in municipalities if not any(l["municipality"] == m for l in places.values())) - if empty: - sys.exit(f"municipalities without a locality: {empty}") + regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in regions.items()} + municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipalities.items()} + regions, municipalities, places = geo.fence(regions, municipalities, places, named) - tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [{"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in sorted(regions.items())]) - tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [{"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in sorted(municipalities.items())]) + tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(regions.items())]) + tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(municipalities.items())]) tsv.write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())]) tsv.write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in places.values() for c in l["codes"]), key=lambda r: r["code"])) tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(named) for s in named[locality]]) diff --git a/data-import/geo-us.py b/data-import/geo-us.py index 7b7297d..b2fa40a 100644 --- a/data-import/geo-us.py +++ b/data-import/geo-us.py @@ -171,12 +171,10 @@ def main(): places = localities(cache, a.min_population, counties) locality_of_zcta = postal_codes(cache, places) named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.streets_per_locality) - places = geo.with_streets(places, named) - kept_counties = {l["municipality"] for l in places.values()} - kept_regions = {counties[c]["region"] for c in kept_counties} + regions, counties, places = geo.fence(regions, counties, places, named) - tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(regions.items()) if r["abbr"] in kept_regions]) - tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(counties.items()) if c["code"] in kept_counties]) + tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(regions.items())]) + tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(counties.items())]) tsv.write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())]) tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], [{"code": z, "locality": l, "addresses": addresses[z]} for z, l in sorted(locality_of_zcta.items()) if addresses[z] and l in places]) tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(named) for s in named[locality]]) diff --git a/data-import/geo.py b/data-import/geo.py index 9d31bc8..f282933 100644 --- a/data-import/geo.py +++ b/data-import/geo.py @@ -72,10 +72,22 @@ def top_streets(count, per_locality, column): return {locality: [{"name": name, "locality": locality, column: n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:per_locality]] for locality, named in of.items()} -def with_streets(places, named): - """The places that named holds streets for; logs each other place as dropped.""" - for key, place in places.items(): - if key not in named: - label = key if key == place["name"] else f"{key} {place['name']}" - print(f"{label}: no streets, dropped", file=sys.stderr) - return {key: place for key, place in places.items() if key in named} +def fence(regions, municipalities, localities, streets): + """Meet the loader's fence that every parent row has a child row: drop each locality streets holds no rows for, then + each municipality and region left without one, logging each drop. regions and municipalities map a code to a row, + and a municipality row names its region under "region". localities map a key to a locality row: name, municipality, + population, lat and lon, besides the columns its country adds. streets maps a locality's key to its street rows. + Returns the kept regions, municipalities and localities.""" + localities = kept(localities, streets, "streets") + municipalities = kept(municipalities, {l["municipality"] for l in localities.values()}, "localities") + regions = kept(regions, {m["region"] for m in municipalities.values()}, "municipalities") + return regions, municipalities, localities + + +def kept(parents, children, kind): + """The parents whose key children holds; logs each other parent as dropped.""" + for key, parent in parents.items(): + if key not in children: + label = key if key == parent["name"] else f"{key} {parent['name']}" + print(f"{label}: no {kind}, dropped", file=sys.stderr) + return {key: parent for key, parent in parents.items() if key in children}