diff --git a/README.md b/README.md index a07b7b5..aed8bf4 100644 --- a/README.md +++ b/README.md @@ -395,7 +395,6 @@ calc.go the {calc()} arithmetic evaluator: parser, eval, validation data.go data loading: fs.FS folders/files -> namespace tree, multi-source merge cmd/fejkdata/ the fejkdata CLI data/ shipped data (JSON), embedded at build: locale folders + a misc folder -format-migration/ converters from the pre-release grammar; delete before the first tag ``` ## License diff --git a/format-migration/convert.py b/format-migration/convert.py deleted file mode 100755 index 073d3ef..0000000 --- a/format-migration/convert.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -"""Rewrite data files from the class-char format grammar to the literal-text one. - - convert.py data/sv_SE/person.json ... - -Old: 0 1 A a are character classes, # escapes. New: text is literal; a run of -0/A/a becomes {digits(n)}/{upper(n)}/{lower(n)}, 1 followed by n zeros becomes -{int(10^n, 10^(n+1)-1)}, a literal brace becomes {{ or }}. -""" -import re -import sys - -RUN = {"0": "{{digits({})}}", "A": "{{upper({})}}", "a": "{{lower({})}}"} - - -def literal(c): - return {"{": "{{", "}": "}}"}.get(c, c) - - -def convert_format(f): - out, i, n = [], 0, len(f) - while i < n: - c = f[i] - if c == "#": - i += 1 - if i < n: - out.append(literal(f[i])) - i += 1 - else: - out.append("#") - elif c == "{": - j = f.find("}", i) - if j < 0: - out.append(f[i:]) - break - out.append(f[i:j + 1]) - i = j + 1 - elif c in RUN: - k = 0 - while i < n and f[i] == c: - k += 1 - i += 1 - out.append(RUN[c].format(k)) - elif c == "1": - i += 1 - k = 0 - while i < n and f[i] == "0": - k += 1 - i += 1 - out.append("{{int({},{})}}".format(10 ** k, 10 ** (k + 1) - 1)) - else: - out.append(literal(c)) - i += 1 - return "".join(out) - - -FORMAT_VALUE = re.compile(r'("format":\s*")((?:[^"\\]|\\.)*)(")') - - -def convert_text(text): - return FORMAT_VALUE.sub(lambda m: m.group(1) + convert_format(m.group(2)) + m.group(3), text) - - -if __name__ == "__main__": - for path in sys.argv[1:]: - with open(path) as fh: - before = fh.read() - after = convert_text(before) - if after != before: - with open(path, "w") as fh: - fh.write(after) - print("converted", path) diff --git a/format-migration/reshape.py b/format-migration/reshape.py deleted file mode 100755 index 81e5d8d..0000000 --- a/format-migration/reshape.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python3 -"""Rewrite data files to the one spelling each shape has. - - reshape.py data/sv_SE/person.json ... - -A one-item choice becomes its item; an object holding only a format becomes that -string; weight 1 and repeat 1 are dropped; a repeated choice item becomes one -item with a weight. -""" -import json -import sys - -OPTIONS = ("format", "weight", "repeat", "separator") -WIDTH = 110 - - -def reshape(n): - if isinstance(n, list): - items = [reshape(x) for x in n] - if len(items) == 1: - return items[0] - out, at = [], {} - for x in items: - key = json.dumps(x, sort_keys=True) - if key in at: - i = at[key] - if isinstance(out[i], str): - out[i] = {"format": out[i], "weight": 2} - elif isinstance(out[i], dict): - out[i]["weight"] = out[i].get("weight", 1) + 1 - else: - out.append(x) - continue - at[key] = len(out) - out.append(x) - return out - if isinstance(n, dict): - d = {k: (v if k in OPTIONS else reshape(v)) for k, v in n.items()} - if d.get("weight") == 1: - del d["weight"] - if d.get("repeat") == 1: - del d["repeat"] - if list(d) == ["format"]: - return d["format"] - return d - return n - - -def scalar(v): - return json.dumps(v, ensure_ascii=False) - - -def one_line(n): - if isinstance(n, dict): - return "{ " + ", ".join(scalar(k) + ": " + one_line(v) for k, v in n.items()) + " }" - if isinstance(n, list): - return "[" + ", ".join(one_line(x) for x in n) + "]" - return scalar(n) - - -def dump(n, indent=0): - pad = " " * indent - if isinstance(n, dict): - line = one_line(n) - if len(pad) + len(line) <= WIDTH and not any(isinstance(v, dict) for v in n.values()): - return line - body = ",\n".join(pad + " " + scalar(k) + ": " + dump(v, indent + 1) for k, v in n.items()) - return "{\n" + body + "\n" + pad + "}" - if isinstance(n, list): - if all(isinstance(x, str) for x in n): - return one_line(n) - line = one_line(n) - if len(pad) + len(line) <= WIDTH: - return line - body = ",\n".join(pad + " " + dump(x, indent + 1) for x in n) - return "[\n" + body + "\n" + pad + "]" - return scalar(n) - - -if __name__ == "__main__": - for path in sys.argv[1:]: - with open(path) as fh: - before = json.load(fh) - after = dump(reshape(before)) + "\n" - with open(path, "w") as fh: - fh.write(after) - print("reshaped", path)