larvit/fejkdata
3
0
Fork 0
Code Pull Requests 1 Actions Activity

Answer architecture round 1: fence keeps a locality holding both a street and a postal code, returns what it dropped so the SE import still exits on a lost kommun and the US one counts its counties, and file item 168 for data-import tests

Browse Source
This commit is contained in:
Lilleman auf Larv
2026-10-09 12:16:33 +02:00
committed by Lillem4n
parent 4f369b961c
commit 71b17d104b
4 changed files with 47 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data-import/geo-se.py
+11 -8
View File
@@ -178,21 +178,24 @@ def main():
sys.exit("set TRAFIKVERKET_API_KEY or pass --key-file")
cache, out = geo.directories(a)
regions, municipalities = scb_codes(cache)
region_names, municipality_names = scb_codes(cache)
population = scb_population(cache)
codes = geonames(cache)
places = localities(codes, scb_tatorter(cache), municipalities, population)
places = localities(codes, scb_tatorter(cache), municipality_names, population)
named = streets(nvdb_segments(cache, key), codes, places, a.streets_per_locality)
regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in regions.items()}
municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipalities.items()}
regions, municipalities, places = geo.fence(regions, municipalities, places, named)
regions = {c: {"code": c, "name": n, "population": population[c], "timezone": TIMEZONE} for c, n in region_names.items()}
municipalities = {c: {"code": c, "name": n, "region": c[:2], "population": population[c]} for c, n in municipality_names.items()}
kept, dropped = geo.fence(regions, municipalities, places, named, {k: l["codes"] for k, l in places.items()})
geo.log_dropped(dropped["locality"], "no street or postal code")
if dropped["municipality"]:
sys.exit(f"municipalities without a locality: {sorted(dropped['municipality'])}")
places = kept["locality"]
tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(regions.items())])
tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(municipalities.items())])
tsv.write(out / "region.tsv", ["code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [m for _, m in sorted(kept["municipality"].items())])
tsv.write(out / "locality.tsv", ["name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
tsv.write(out / "postal-code.tsv", ["code", "locality"], sorted(({"code": f"{c[:3]} {c[3:]}", "locality": l["name"]} for l in places.values() for c in l["codes"]), key=lambda r: r["code"]))
tsv.write(out / "street.tsv", ["name", "locality", "segments"], [s for locality in sorted(named) for s in named[locality]])
if __name__ == "__main__":
main()
data-import/geo-us.py
+11 -5
View File
@@ -171,14 +171,20 @@ def main():
places = localities(cache, a.min_population, counties)
locality_of_zcta = postal_codes(cache, places)
named, addresses = streets(cache, sorted({l["municipality"] for l in places.values()}), locality_of_zcta, a.streets_per_locality)
regions, counties, places = geo.fence(regions, counties, places, named)
zctas = collections.defaultdict(list)
for zcta, locality in sorted(locality_of_zcta.items()):
if addresses[zcta]:
zctas[locality].append(zcta)
kept, dropped = geo.fence(regions, counties, places, named, zctas)
geo.log_dropped(dropped["locality"], "no street or postal code")
print(f"counties without a place, dropped: {len(dropped['municipality'])}; states: {sorted(dropped['region'])}", file=sys.stderr)
places = kept["locality"]
tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(regions.items())])
tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(counties.items())])
tsv.write(out / "region.tsv", ["abbr", "code", "name", "population", "timezone"], [r for _, r in sorted(kept["region"].items())])
tsv.write(out / "municipality.tsv", ["code", "name", "region", "population"], [c for _, c in sorted(kept["municipality"].items())])
tsv.write(out / "locality.tsv", ["code", "name", "municipality", "population", "lat", "lon"], [l for _, l in sorted(places.items())])
tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], [{"code": z, "locality": l, "addresses": addresses[z]} for z, l in sorted(locality_of_zcta.items()) if addresses[z] and l in places])
tsv.write(out / "postal-code.tsv", ["code", "locality", "addresses"], sorted(({"code": z, "locality": l, "addresses": addresses[z]} for l in places for z in zctas[l]), key=lambda r: r["code"]))
tsv.write(out / "street.tsv", ["name", "locality", "addresses"], [s for locality in sorted(named) for s in named[locality]])
if __name__ == "__main__":
main()
data-import/geo.py
+19 -17
View File
@@ -72,22 +72,24 @@ def top_streets(count, per_locality, column):
return {locality: [{"name": name, "locality": locality, column: n} for n, name in sorted(named, key=lambda s: (-s[0], s[1]))[:per_locality]] for locality, named in of.items()}
def fence(regions, municipalities, localities, streets):
"""Meet the loader's fence that every parent row has a child row: drop each locality streets holds no rows for, then
each municipality and region left without one, logging each drop. regions and municipalities map a code to a row,
and a municipality row names its region under "region". localities map a key to a locality row: name, municipality,
population, lat and lon, besides the columns its country adds. streets maps a locality's key to its street rows.
Returns the kept regions, municipalities and localities."""
localities = kept(localities, streets, "streets")
municipalities = kept(municipalities, {l["municipality"] for l in localities.values()}, "localities")
regions = kept(regions, {m["region"] for m in municipalities.values()}, "municipalities")
return regions, municipalities, localities
def fence(regions, municipalities, localities, streets, postal_codes):
"""Meet the loader's fence that every parent row has a child row: keep each locality both streets and postal_codes
hold a key for, then each municipality and region left with a child. regions and municipalities map a code to a
row, and a municipality row names its region under "region". localities map a key to a locality row: name,
municipality, population, lat and lon, besides the columns its country adds. Returns the kept rows, then the dropped
rows, each a dict of the three tables by name."""
kept = {"locality": {k: l for k, l in localities.items() if k in streets and k in postal_codes}}
parents = {l["municipality"] for l in kept["locality"].values()}
kept["municipality"] = {c: m for c, m in municipalities.items() if c in parents}
parents = {m["region"] for m in kept["municipality"].values()}
kept["region"] = {c: r for c, r in regions.items() if c in parents}
every = {"locality": localities, "municipality": municipalities, "region": regions}
dropped = {table: {k: row for k, row in rows.items() if k not in kept[table]} for table, rows in every.items()}
return kept, dropped
def kept(parents, children, kind):
"""The parents whose key children holds; logs each other parent as dropped."""
for key, parent in parents.items():
if key not in children:
label = key if key == parent["name"] else f"{key} {parent['name']}"
print(f"{label}: no {kind}, dropped", file=sys.stderr)
return {key: parent for key, parent in parents.items() if key in children}
def log_dropped(rows, why):
"""Logs each row as dropped for why, by its key and its name where the two differ."""
for key, row in rows.items():
label = key if key == row["name"] else f"{key} {row['name']}"
print(f"{label}: {why}, dropped", file=sys.stderr)