Name the layout without the quotes the user wrote, return a download written on the last attempt, and put the link fences' description back on linkTables
This commit is contained in:
@@ -17,8 +17,8 @@ import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import source
|
||||
import xlsx
|
||||
import tsv
|
||||
import xlsx
|
||||
|
||||
CODES = "https://www.scb.se/contentassets/7a89e48960f741e08918e489ea36354a/kommunlankod-2026.xlsx"
|
||||
POPULATION = "https://api.scb.se/OV0104/v1/doris/sv/ssd/START/BE/BE0101/BE0101A/BefolkningNy"
|
||||
@@ -39,6 +39,8 @@ CACHE = Path(__file__).resolve().parent / "cache"
|
||||
TIMEZONE = "Europe/Stockholm"
|
||||
ONE_POSITION = {"Stockholm", "Göteborg", "Malmö"}
|
||||
UNMATCHED_POPULATION = 200
|
||||
|
||||
|
||||
def scb_codes(cache):
|
||||
regions, municipalities = {}, {}
|
||||
for cells in xlsx.rows(source.fetch(CODES, cache, "kommunlankod.xlsx", magic=b"PK")):
|
||||
|
||||
@@ -10,8 +10,8 @@ import re
|
||||
from pathlib import Path
|
||||
|
||||
import source
|
||||
import xlsx
|
||||
import tsv
|
||||
import xlsx
|
||||
|
||||
SOURCE = "https://www.scb.se/contentassets/9fe7dbb460994c72b835163dbc491ef9/namn-med-minst-tva-barare-31-december-2022.xlsx"
|
||||
OUT = Path(__file__).resolve().parent.parent / "data" / "sv_SE"
|
||||
|
||||
@@ -11,9 +11,9 @@ def fetch(source, cache, name, magic=b"", data=None, headers=None):
|
||||
if not re.match(r"^https?://", source):
|
||||
return Path(source).read_bytes()
|
||||
path = Path(cache) / name
|
||||
if path.exists():
|
||||
return path.read_bytes()
|
||||
for attempt in range(1, 6):
|
||||
if path.exists():
|
||||
return path.read_bytes()
|
||||
req = urllib.request.Request(source, data=data, headers={"User-Agent": "fejkdata data-import", **(headers or {})})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=600) as r:
|
||||
@@ -23,6 +23,7 @@ def fetch(source, cache, name, magic=b"", data=None, headers=None):
|
||||
if body and body.startswith(magic) and b"Request Rejected" not in body[:512]:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(body)
|
||||
elif attempt < 5:
|
||||
return body
|
||||
if attempt < 5:
|
||||
time.sleep(10 * attempt)
|
||||
sys.exit(f"{source}: no valid download in 5 attempts")
|
||||
|
||||
Reference in New Issue
Block a user