From edce645f343087cf531d75d08601f71f90035de8 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 20 Sep 2026 20:37:24 +0200 Subject: [PATCH 01/10] Expect RFC 5424's severities as a table, selectable by code or name --- data_test.go | 2 ++ table_test.go | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/data_test.go b/data_test.go index 7e1b0b2..84c4eaa 100644 --- a/data_test.go +++ b/data_test.go @@ -119,6 +119,8 @@ func TestShippedMiscReferenceData(t *testing.T) { "protocol": regexp.MustCompile(`^\S+( \S+)*$`), "protocol.name": regexp.MustCompile(`\S`), "protocol.number": regexp.MustCompile(`^\d{1,3}$`), + "loglevel": regexp.MustCompile(`^(Alert|Critical|Debug|Emergency|Error|Informational|Notice|Warning)$`), + "loglevel.code": regexp.MustCompile(`^[0-7]$`), "tld": regexp.MustCompile(`^\.[a-z0-9]([a-z0-9-]*[a-z0-9])?$`), "tld.type": regexp.MustCompile(`^(country-code|generic|generic-restricted|infrastructure|sponsored)$`), "tld.unicode": regexp.MustCompile(`^\.\S+$`), diff --git a/table_test.go b/table_test.go index 6e0e348..a85cd7e 100644 --- a/table_test.go +++ b/table_test.go @@ -725,6 +725,15 @@ func TestShippedTables(t *testing.T) { "tld[.рф]": ".xn--p1ai", "tld[.xn--p1ai].unicode": ".рф", "territory[NP].timezone.offset": "+05:45", + "loglevel[0]": "Emergency", + "loglevel[1]": "Alert", + "loglevel[2]": "Critical", + "loglevel[3]": "Error", + "loglevel[4]": "Warning", + "loglevel[5]": "Notice", + "loglevel[6]": "Informational", + "loglevel[7]": "Debug", + "loglevel[Informational].code": "6", } { if got := fake(t, f, path); got != want { t.Errorf("Fake(%q) = %q, want %q", path, got, want) -- 2.52.0 From 474f73444bae6508199822650201e82d8b5a9b44 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 20 Sep 2026 20:39:21 +0200 Subject: [PATCH 02/10] Read misc.loglevel from RFC 5424's severity table --- CHANGELOG.md | 3 +++ DATA-LICENSES.md | 1 + README.md | 25 ++++++++++++++------- data-import/loglevel.py | 49 +++++++++++++++++++++++++++++++++++++++++ data/misc/loglevel.json | 1 + data/misc/loglevel.tsv | 9 ++++++++ todo.md | 1 - 7 files changed, 80 insertions(+), 9 deletions(-) create mode 100644 data-import/loglevel.py create mode 100644 data/misc/loglevel.json create mode 100644 data/misc/loglevel.tsv diff --git a/CHANGELOG.md b/CHANGELOG.md index 005dec6..04238bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,9 @@ replacement, and each removed path, column or flag. leading dot, `.se` and `.xn--p1ai`, the register's `type` beside it, and a `unicode` column holding the form the register displays, so `misc.tld[.рф]` selects the row `.xn--p1ai` renders. A delegation the register marks "Not assigned" does not ship. +- `misc.loglevel` is RFC 5424's severity table: the eight levels a syslog PRI encodes, + keyed by the numerical `code` and rendering the `severity` the RFC spells, so + `misc.loglevel[3]` and `misc.loglevel[Error]` select one row. - `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`, `locality`, `postal-code` and `street`, weighted by population and address counts and built from SCB, GeoNames, Trafikverket NVDB and the US Census Bureau, and an diff --git a/DATA-LICENSES.md b/DATA-LICENSES.md index 033cbb1..26f4bfa 100644 --- a/DATA-LICENSES.md +++ b/DATA-LICENSES.md @@ -19,6 +19,7 @@ Every shipped dataset, its source, its licence and the attribution it asks for. | `misc/httpmethod.tsv` | [IANA HTTP Method Registry](https://www.iana.org/assignments/http-methods/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpmethod.py` | | `misc/httpstatus.tsv` | [IANA HTTP Status Code Registry](https://www.iana.org/assignments/http-status-codes/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpstatus.py` | | `misc/language.tsv` | [datasets/language-codes](https://github.com/datasets/language-codes), the [Library of Congress](https://www.loc.gov/standards/iso639-2/) ISO 639-2 register | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/language.py` | +| `misc/loglevel.tsv` | [RFC 5424](https://www.rfc-editor.org/rfc/rfc5424.txt) Table 2, the syslog message severities | facts | none required | `data-import/loglevel.py` | | `misc/mimetype.tsv` | [mime-db](https://github.com/jshttp/mime-db), the IANA media type registry with filename extensions | [MIT](https://github.com/jshttp/mime-db/blob/master/LICENSE) | "Copyright (c) 2014 Jonathan Ong, Copyright (c) 2015-2022 Douglas Christopher Wilson" | `data-import/mimetype.py` | | `misc/port.tsv` | [IANA Service Name and Transport Protocol Port Number Registry](https://www.iana.org/assignments/service-names-port-numbers/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/port.py` | | `misc/protocol.tsv` | [IANA Protocol Numbers](https://www.iana.org/assignments/protocol-numbers/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/protocol.py` | diff --git a/README.md b/README.md index b0756fc..8305d43 100644 --- a/README.md +++ b/README.md @@ -163,14 +163,14 @@ Each locale carries `address`, `color`, `company`, `date`, `email`, `first-name` `personnummer` and `samordningsnummer`, `en_US` adds `ssn` and `itin`. `misc` carries `car`, `coordinate`, `creditcard` (Luhn-valid), `currency` (ISO 4217), `datetime` (RFC 3339), `emoji`, `httpmethod`, `httpstatus`, `language` (ISO 639-1, -with its 639-2/T code), `mac`, `mimetype`, `objectid`, `port`, `protocol`, -`territory` (ISO 3166-1), `timezone` (IANA), `tld` (IANA root zone), `useragent` and -`uuid` (v4). Many carry sub-fields — `misc.currency.symbol`, `misc.territory.alpha2`, -`misc.httpstatus.code` — which `--list` shows. `car`, `currency`, `httpmethod`, -`httpstatus`, `language`, `mimetype`, `port`, `protocol`, `territory`, `timezone`, -`tld` and `useragent` are [tables](#table), so `misc.territory[SE].capital` and -`misc.currency[Euro].symbol` select a row; `car` and `useragent` carry no key or name, -so they are drawn from rather than selected in. +with its 639-2/T code), `loglevel` (RFC 5424), `mac`, `mimetype`, `objectid`, `port`, +`protocol`, `territory` (ISO 3166-1), `timezone` (IANA), `tld` (IANA root zone), +`useragent` and `uuid` (v4). Many carry sub-fields — `misc.currency.symbol`, +`misc.territory.alpha2`, `misc.httpstatus.code` — which `--list` shows. `car`, +`currency`, `httpmethod`, `httpstatus`, `language`, `loglevel`, `mimetype`, `port`, +`protocol`, `territory`, `timezone`, `tld` and `useragent` are [tables](#table), so +`misc.territory[SE].capital` and `misc.currency[Euro].symbol` select a row; `car` and +`useragent` carry no key or name, so they are drawn from rather than selected in. `misc.httpmethod`, `misc.protocol` and `misc.port` are IANA's registries. `misc.httpmethod` is the eight methods RFC 9110 defines and PATCH; the register's @@ -191,6 +191,9 @@ its `unicode` the form the register displays, `.рф` for `.xn--p1ai` and the ke for an ASCII one, which selects the row too, so `misc.tld[.рф]` renders `.xn--p1ai`. A draw spans the whole zone, `.arpa` and `.zuerich` alike, so pin `misc.tld[.com]` where a fixture needs one a reader recognises. +`misc.loglevel` is RFC 5424's eight severities, keyed by the numerical code a syslog +PRI carries and named by the severity, so `misc.loglevel[3]` and `misc.loglevel[Error]` +are one row and `misc.loglevel.code` the number beside the name. [`DATA-LICENSES.md`](DATA-LICENSES.md) names each table's source and licence. `misc.timezone` is every zone tzdb gives a shipped territory, from one apiece for most @@ -1230,6 +1233,11 @@ the Development section below, and who ships a register the four above then draw every value names a row instead. A territory the register records no state for stands alone, which is `EH` alone, and naming one for it would be a claim fejkdata has no business making. +- **`misc.loglevel` is a table keyed by the code, not a list of names.** A syslog + message carries the severity as the number inside its PRI, so the code and the name + are one fact and goal 1 draws them together. The `emerg`, `err` and `info` an + rsyslog config takes are BSD's spellings, which RFC 5424 writes nowhere, so shipping + them would be the hand-written mapping goal 10 keeps out of a sourced table. - **`misc.tld` keys carry the leading dot, where other tables key on a bare code.** The register spells a TLD `.se` and `misc.territory.tld` already ships it so, which a bare key would make two spellings of one fact; `{/misc.tld}` also composes onto a @@ -1339,6 +1347,7 @@ docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/geo-u docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/httpmethod.py docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/httpstatus.py docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/language.py +docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/loglevel.py docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/mimetype.py docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/names-se.py docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/names-us.py diff --git a/data-import/loglevel.py b/data-import/loglevel.py new file mode 100644 index 0000000..439e067 --- /dev/null +++ b/data-import/loglevel.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Rebuild data/misc/loglevel.tsv from RFC 5424's severity table. + + data-import/loglevel.py [--source URL_OR_FILE] [--cache DIR] [--out FILE] + +Table 2 is the whole set, read between the facility table's caption and its own. The +RFC describes both tables for informational purposes, so a severity is what an +originator writes rather than something a collector rejects. +""" +import argparse +import re +import sys +from pathlib import Path + +import source +import tsv + +SOURCE = "https://www.rfc-editor.org/rfc/rfc5424.txt" +OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "loglevel.tsv" +CACHE = Path(__file__).resolve().parent / "cache" +COLUMNS = ["code", "severity"] +TABLE = re.compile(r"Table 1\.\s+Syslog Message Facilities(.*?)Table 2\.\s+Syslog Message Severities", re.S) +ROW = re.compile(r"^ +(\d) +([A-Z][a-z]+): ", re.M) +EXPECTED = 8 + + +def rows(text): + table = TABLE.search(text) + if not table: + sys.exit("the RFC holds no text between the facility and the severity caption; its tables have moved") + for code, severity in ROW.findall(table.group(1)): + yield {"code": code, "severity": severity} + + +def main(): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument("--cache", default=str(CACHE)) + p.add_argument("--source", default=SOURCE) + p.add_argument("--out", default=str(OUT)) + a = p.parse_args() + table = sorted(rows(source.fetch(a.source, a.cache, "rfc5424.txt").decode("utf-8")), key=lambda r: r["code"]) + codes = [r["code"] for r in table] + if codes != [str(i) for i in range(EXPECTED)]: + sys.exit(f"the severity table numbers {codes}, not 0 through {EXPECTED - 1}") + tsv.write(a.out, COLUMNS, table) + + +if __name__ == "__main__": + main() diff --git a/data/misc/loglevel.json b/data/misc/loglevel.json new file mode 100644 index 0000000..9e9d3f1 --- /dev/null +++ b/data/misc/loglevel.json @@ -0,0 +1 @@ +{ "format": "{severity}", "rows": "loglevel.tsv", "key": "code", "name": "severity" } diff --git a/data/misc/loglevel.tsv b/data/misc/loglevel.tsv new file mode 100644 index 0000000..8ba335c --- /dev/null +++ b/data/misc/loglevel.tsv @@ -0,0 +1,9 @@ +code severity +0 Emergency +1 Alert +2 Critical +3 Error +4 Warning +5 Notice +6 Informational +7 Debug diff --git a/todo.md b/todo.md index 514a1f8..6db496a 100644 --- a/todo.md +++ b/todo.md @@ -49,7 +49,6 @@ by a `data-import/` script, as README goal 10 asks. | Category | Shape | Source | Licence | |---|---|---|---| -| `loglevel` | c | RFC 5424 severities | facts | | `ip` v4 documentation ranges, private, CIDR; `ipv6` `2001:db8::/32`; `mac` locally administered | t | RFC 5737, 1918, 9637 | facts | | `creditcard` per network: IIN, length, CVV, expiry | T+t | network rules, `{luhn()}` | facts | | `bic`, `isin`, `cusip` | t | structure rules | facts | -- 2.52.0 From d081ec85d0a47a6c2d4077b2b968d2d4f177636e Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 20 Sep 2026 20:39:52 +0200 Subject: [PATCH 03/10] Re-pin the shipped shape for misc.loglevel --- testdata/shipped_shape.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testdata/shipped_shape.txt b/testdata/shipped_shape.txt index d14f80c..b882ed4 100644 --- a/testdata/shipped_shape.txt +++ b/testdata/shipped_shape.txt @@ -240,6 +240,9 @@ misc.language format "{name}" key code name name misc.language.code string misc.language.code3 string misc.language.name string +misc.loglevel format "{severity}" key code name severity +misc.loglevel.code string +misc.loglevel.severity string misc.mac format "{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}" misc.mimetype format "{type}" key type name ext misc.mimetype.ext string -- 2.52.0 From 839b7bd164583adbc9f7e4a158d6acaa3c844288 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 20 Sep 2026 20:47:19 +0200 Subject: [PATCH 04/10] Cut the importer's rationale, pin the name selection the docs show, and spell the PRI alike --- README.md | 4 ++-- data-import/loglevel.py | 4 +--- table_test.go | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8305d43..a4b4170 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ for an ASCII one, which selects the row too, so `misc.tld[.рф]` renders `.xn-- draw spans the whole zone, `.arpa` and `.zuerich` alike, so pin `misc.tld[.com]` where a fixture needs one a reader recognises. `misc.loglevel` is RFC 5424's eight severities, keyed by the numerical code a syslog -PRI carries and named by the severity, so `misc.loglevel[3]` and `misc.loglevel[Error]` +PRI encodes and named by the severity, so `misc.loglevel[3]` and `misc.loglevel[Error]` are one row and `misc.loglevel.code` the number beside the name. [`DATA-LICENSES.md`](DATA-LICENSES.md) names each table's source and licence. @@ -1234,7 +1234,7 @@ the Development section below, and who ships a register the four above then draw stands alone, which is `EH` alone, and naming one for it would be a claim fejkdata has no business making. - **`misc.loglevel` is a table keyed by the code, not a list of names.** A syslog - message carries the severity as the number inside its PRI, so the code and the name + message carries the severity as the number its PRI encodes, so the code and the name are one fact and goal 1 draws them together. The `emerg`, `err` and `info` an rsyslog config takes are BSD's spellings, which RFC 5424 writes nowhere, so shipping them would be the hand-written mapping goal 10 keeps out of a sourced table. diff --git a/data-import/loglevel.py b/data-import/loglevel.py index 439e067..ed8cafb 100644 --- a/data-import/loglevel.py +++ b/data-import/loglevel.py @@ -3,9 +3,7 @@ data-import/loglevel.py [--source URL_OR_FILE] [--cache DIR] [--out FILE] -Table 2 is the whole set, read between the facility table's caption and its own. The -RFC describes both tables for informational purposes, so a severity is what an -originator writes rather than something a collector rejects. +Table 2 is the whole set, read between the facility table's caption and its own. """ import argparse import re diff --git a/table_test.go b/table_test.go index a85cd7e..62ffb68 100644 --- a/table_test.go +++ b/table_test.go @@ -733,7 +733,7 @@ func TestShippedTables(t *testing.T) { "loglevel[5]": "Notice", "loglevel[6]": "Informational", "loglevel[7]": "Debug", - "loglevel[Informational].code": "6", + "loglevel[Error].code": "3", } { if got := fake(t, f, path); got != want { t.Errorf("Fake(%q) = %q, want %q", path, got, want) -- 2.52.0 From 621e6f45484f61292e7376eee1edabf818dfecea Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 20 Sep 2026 20:52:07 +0200 Subject: [PATCH 05/10] Drop the loglevel decision's false goal-10 claim --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a4b4170..94bf57e 100644 --- a/README.md +++ b/README.md @@ -1235,9 +1235,8 @@ the Development section below, and who ships a register the four above then draw fejkdata has no business making. - **`misc.loglevel` is a table keyed by the code, not a list of names.** A syslog message carries the severity as the number its PRI encodes, so the code and the name - are one fact and goal 1 draws them together. The `emerg`, `err` and `info` an - rsyslog config takes are BSD's spellings, which RFC 5424 writes nowhere, so shipping - them would be the hand-written mapping goal 10 keeps out of a sourced table. + are one fact and goal 1 draws them together; a flat list of names would carry + neither the code nor a selector reaching it. - **`misc.tld` keys carry the leading dot, where other tables key on a bare code.** The register spells a TLD `.se` and `misc.territory.tld` already ships it so, which a bare key would make two spellings of one fact; `{/misc.tld}` also composes onto a -- 2.52.0 From 5715e516aa078d637a7f8e594df3a1c0986ae723 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 20 Sep 2026 21:19:11 +0200 Subject: [PATCH 06/10] Expect the syslog keyword misc.loglevel renders, beside the RFC severity --- data_test.go | 3 ++- table_test.go | 19 ++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/data_test.go b/data_test.go index 84c4eaa..7dbe162 100644 --- a/data_test.go +++ b/data_test.go @@ -119,8 +119,9 @@ func TestShippedMiscReferenceData(t *testing.T) { "protocol": regexp.MustCompile(`^\S+( \S+)*$`), "protocol.name": regexp.MustCompile(`\S`), "protocol.number": regexp.MustCompile(`^\d{1,3}$`), - "loglevel": regexp.MustCompile(`^(Alert|Critical|Debug|Emergency|Error|Informational|Notice|Warning)$`), + "loglevel": regexp.MustCompile(`^(alert|crit|debug|emerg|err|info|notice|warning)$`), "loglevel.code": regexp.MustCompile(`^[0-7]$`), + "loglevel.severity": regexp.MustCompile(`^(Alert|Critical|Debug|Emergency|Error|Informational|Notice|Warning)$`), "tld": regexp.MustCompile(`^\.[a-z0-9]([a-z0-9-]*[a-z0-9])?$`), "tld.type": regexp.MustCompile(`^(country-code|generic|generic-restricted|infrastructure|sponsored)$`), "tld.unicode": regexp.MustCompile(`^\.\S+$`), diff --git a/table_test.go b/table_test.go index 62ffb68..66529db 100644 --- a/table_test.go +++ b/table_test.go @@ -725,15 +725,16 @@ func TestShippedTables(t *testing.T) { "tld[.рф]": ".xn--p1ai", "tld[.xn--p1ai].unicode": ".рф", "territory[NP].timezone.offset": "+05:45", - "loglevel[0]": "Emergency", - "loglevel[1]": "Alert", - "loglevel[2]": "Critical", - "loglevel[3]": "Error", - "loglevel[4]": "Warning", - "loglevel[5]": "Notice", - "loglevel[6]": "Informational", - "loglevel[7]": "Debug", - "loglevel[Error].code": "3", + "loglevel[0]": "emerg", + "loglevel[1]": "alert", + "loglevel[2]": "crit", + "loglevel[3]": "err", + "loglevel[4]": "warning", + "loglevel[5]": "notice", + "loglevel[6]": "info", + "loglevel[7]": "debug", + "loglevel[err].code": "3", + "loglevel[6].severity": "Informational", } { if got := fake(t, f, path); got != want { t.Errorf("Fake(%q) = %q, want %q", path, got, want) -- 2.52.0 From 81cf04ae8b284387168d69d001bf03b30d97357c Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 20 Sep 2026 21:20:44 +0200 Subject: [PATCH 07/10] Render the syslog keyword POSIX publishes, keeping RFC 5424's severity beside it --- CHANGELOG.md | 6 +++--- DATA-LICENSES.md | 2 +- README.md | 23 ++++++++++++---------- data-import/loglevel.py | 42 +++++++++++++++++++++++++++++++---------- data/misc/loglevel.json | 2 +- data/misc/loglevel.tsv | 18 +++++++++--------- 6 files changed, 59 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04238bc..de8b1c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,9 +53,9 @@ replacement, and each removed path, column or flag. leading dot, `.se` and `.xn--p1ai`, the register's `type` beside it, and a `unicode` column holding the form the register displays, so `misc.tld[.рф]` selects the row `.xn--p1ai` renders. A delegation the register marks "Not assigned" does not ship. -- `misc.loglevel` is RFC 5424's severity table: the eight levels a syslog PRI encodes, - keyed by the numerical `code` and rendering the `severity` the RFC spells, so - `misc.loglevel[3]` and `misc.loglevel[Error]` select one row. +- `misc.loglevel` is the eight syslog severities: the `keyword` a configuration writes, + `err` and `info`, keyed by the numerical `code` a PRI encodes and selectable by + either, with the `severity` RFC 5424 spells beside it. - `geo.SE` and `geo.US`: five linked tables per country, `region`, `municipality`, `locality`, `postal-code` and `street`, weighted by population and address counts and built from SCB, GeoNames, Trafikverket NVDB and the US Census Bureau, and an diff --git a/DATA-LICENSES.md b/DATA-LICENSES.md index 26f4bfa..48491a9 100644 --- a/DATA-LICENSES.md +++ b/DATA-LICENSES.md @@ -19,7 +19,7 @@ Every shipped dataset, its source, its licence and the attribution it asks for. | `misc/httpmethod.tsv` | [IANA HTTP Method Registry](https://www.iana.org/assignments/http-methods/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpmethod.py` | | `misc/httpstatus.tsv` | [IANA HTTP Status Code Registry](https://www.iana.org/assignments/http-status-codes/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpstatus.py` | | `misc/language.tsv` | [datasets/language-codes](https://github.com/datasets/language-codes), the [Library of Congress](https://www.loc.gov/standards/iso639-2/) ISO 639-2 register | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/language.py` | -| `misc/loglevel.tsv` | [RFC 5424](https://www.rfc-editor.org/rfc/rfc5424.txt) Table 2, the syslog message severities | facts | none required | `data-import/loglevel.py` | +| `misc/loglevel.tsv` | [RFC 5424](https://www.rfc-editor.org/rfc/rfc5424.txt) Table 2, the syslog message severities; keywords from [POSIX](https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/syslog.h.html) `` | facts | none required | `data-import/loglevel.py` | | `misc/mimetype.tsv` | [mime-db](https://github.com/jshttp/mime-db), the IANA media type registry with filename extensions | [MIT](https://github.com/jshttp/mime-db/blob/master/LICENSE) | "Copyright (c) 2014 Jonathan Ong, Copyright (c) 2015-2022 Douglas Christopher Wilson" | `data-import/mimetype.py` | | `misc/port.tsv` | [IANA Service Name and Transport Protocol Port Number Registry](https://www.iana.org/assignments/service-names-port-numbers/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/port.py` | | `misc/protocol.tsv` | [IANA Protocol Numbers](https://www.iana.org/assignments/protocol-numbers/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/protocol.py` | diff --git a/README.md b/README.md index 94bf57e..e487bf6 100644 --- a/README.md +++ b/README.md @@ -163,9 +163,9 @@ Each locale carries `address`, `color`, `company`, `date`, `email`, `first-name` `personnummer` and `samordningsnummer`, `en_US` adds `ssn` and `itin`. `misc` carries `car`, `coordinate`, `creditcard` (Luhn-valid), `currency` (ISO 4217), `datetime` (RFC 3339), `emoji`, `httpmethod`, `httpstatus`, `language` (ISO 639-1, -with its 639-2/T code), `loglevel` (RFC 5424), `mac`, `mimetype`, `objectid`, `port`, -`protocol`, `territory` (ISO 3166-1), `timezone` (IANA), `tld` (IANA root zone), -`useragent` and `uuid` (v4). Many carry sub-fields — `misc.currency.symbol`, +with its 639-2/T code), `loglevel` (syslog severity), `mac`, `mimetype`, `objectid`, +`port`, `protocol`, `territory` (ISO 3166-1), `timezone` (IANA), `tld` (IANA root +zone), `useragent` and `uuid` (v4). Many carry sub-fields — `misc.currency.symbol`, `misc.territory.alpha2`, `misc.httpstatus.code` — which `--list` shows. `car`, `currency`, `httpmethod`, `httpstatus`, `language`, `loglevel`, `mimetype`, `port`, `protocol`, `territory`, `timezone`, `tld` and `useragent` are [tables](#table), so @@ -191,9 +191,10 @@ its `unicode` the form the register displays, `.рф` for `.xn--p1ai` and the ke for an ASCII one, which selects the row too, so `misc.tld[.рф]` renders `.xn--p1ai`. A draw spans the whole zone, `.arpa` and `.zuerich` alike, so pin `misc.tld[.com]` where a fixture needs one a reader recognises. -`misc.loglevel` is RFC 5424's eight severities, keyed by the numerical code a syslog -PRI encodes and named by the severity, so `misc.loglevel[3]` and `misc.loglevel[Error]` -are one row and `misc.loglevel.code` the number beside the name. +`misc.loglevel` is the eight syslog severities, rendering the keyword a configuration +writes, `err` and `info`, keyed by the numerical code a PRI encodes and selectable by +either, so `misc.loglevel[3]` and `misc.loglevel[err]` are one row. +`misc.loglevel.severity` is the name RFC 5424 spells, `Error` beside `err`. [`DATA-LICENSES.md`](DATA-LICENSES.md) names each table's source and licence. `misc.timezone` is every zone tzdb gives a shipped territory, from one apiece for most @@ -1233,10 +1234,12 @@ the Development section below, and who ships a register the four above then draw every value names a row instead. A territory the register records no state for stands alone, which is `EH` alone, and naming one for it would be a claim fejkdata has no business making. -- **`misc.loglevel` is a table keyed by the code, not a list of names.** A syslog - message carries the severity as the number its PRI encodes, so the code and the name - are one fact and goal 1 draws them together; a flat list of names would carry - neither the code nor a selector reaching it. +- **`misc.loglevel` is a table keyed by the code, rendering POSIX's keyword.** A flat + list of names carries neither the code a PRI encodes nor a selector reaching it, and + goal 1 draws the two as one fact. RFC 5424 spells the severity `Informational`, + which no syslog, PSR-3 or `slog` checker accepts, so the shipped spelling is the one + a configuration writes, `info`, and the RFC's name stays the `severity` column. The + code and the keyword take the two selector slots, so `misc.loglevel[Error]` misses. - **`misc.tld` keys carry the leading dot, where other tables key on a bare code.** The register spells a TLD `.se` and `misc.territory.tld` already ships it so, which a bare key would make two spellings of one fact; `{/misc.tld}` also composes onto a diff --git a/data-import/loglevel.py b/data-import/loglevel.py index ed8cafb..67f2242 100644 --- a/data-import/loglevel.py +++ b/data-import/loglevel.py @@ -1,11 +1,15 @@ #!/usr/bin/env python3 -"""Rebuild data/misc/loglevel.tsv from RFC 5424's severity table. +"""Rebuild data/misc/loglevel.tsv from RFC 5424 and POSIX . - data-import/loglevel.py [--source URL_OR_FILE] [--cache DIR] [--out FILE] + data-import/loglevel.py [--source URL_OR_FILE] [--posix URL_OR_FILE] [--cache DIR] [--out FILE] -Table 2 is the whole set, read between the facility table's caption and its own. +Table 2 is the whole set, read between the facility table's caption and its own. The +keyword is POSIX's severity macro less its `LOG_` prefix, lowercased, taken in the +order POSIX lists them, which `LOG_UPTO` states is the severity order; each must be a +prefix of the severity RFC 5424 numbers alike, so the two sources pin one pairing. """ import argparse +import html import re import sys from pathlib import Path @@ -14,33 +18,51 @@ import source import tsv SOURCE = "https://www.rfc-editor.org/rfc/rfc5424.txt" +POSIX = "https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/syslog.h.html" OUT = Path(__file__).resolve().parent.parent / "data" / "misc" / "loglevel.tsv" CACHE = Path(__file__).resolve().parent / "cache" -COLUMNS = ["code", "severity"] +COLUMNS = ["code", "keyword", "severity"] TABLE = re.compile(r"Table 1\.\s+Syslog Message Facilities(.*?)Table 2\.\s+Syslog Message Severities", re.S) ROW = re.compile(r"^ +(\d) +([A-Z][a-z]+): ", re.M) +SEVERITIES = re.compile(r"severity level portion of the.*?macros:(.*?)The following shall be declared as functions", re.S) +MACRO = re.compile(r"\bLOG_([A-Z]+)\b") EXPECTED = 8 -def rows(text): +def severities(text): table = TABLE.search(text) if not table: sys.exit("the RFC holds no text between the facility and the severity caption; its tables have moved") - for code, severity in ROW.findall(table.group(1)): - yield {"code": code, "severity": severity} + return ROW.findall(table.group(1)) + + +def keywords(page): + section = SEVERITIES.search(html.unescape(re.sub(r"<[^>]+>", "", page))) + if not section: + sys.exit("POSIX spells no severity list between its own caption and the function declarations; the page has moved") + return [m.lower() for m in MACRO.findall(section.group(1))] def main(): p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) p.add_argument("--cache", default=str(CACHE)) p.add_argument("--source", default=SOURCE) + p.add_argument("--posix", default=POSIX) p.add_argument("--out", default=str(OUT)) a = p.parse_args() - table = sorted(rows(source.fetch(a.source, a.cache, "rfc5424.txt").decode("utf-8")), key=lambda r: r["code"]) - codes = [r["code"] for r in table] + table = sorted(severities(source.fetch(a.source, a.cache, "rfc5424.txt").decode("utf-8"))) + codes = [code for code, _ in table] if codes != [str(i) for i in range(EXPECTED)]: sys.exit(f"the severity table numbers {codes}, not 0 through {EXPECTED - 1}") - tsv.write(a.out, COLUMNS, table) + words = keywords(source.fetch(a.posix, a.cache, "posix-syslog.h.html").decode("utf-8")) + if len(words) != EXPECTED: + sys.exit(f"POSIX lists {len(words)} severity macros, not {EXPECTED}: {words}") + rows = [] + for (code, severity), keyword in zip(table, words): + if not severity.lower().startswith(keyword): + sys.exit(f"severity {code} is {severity} in the RFC and {keyword} in POSIX, which is no prefix of it") + rows.append({"code": code, "keyword": keyword, "severity": severity}) + tsv.write(a.out, COLUMNS, rows) if __name__ == "__main__": diff --git a/data/misc/loglevel.json b/data/misc/loglevel.json index 9e9d3f1..1144d26 100644 --- a/data/misc/loglevel.json +++ b/data/misc/loglevel.json @@ -1 +1 @@ -{ "format": "{severity}", "rows": "loglevel.tsv", "key": "code", "name": "severity" } +{ "format": "{keyword}", "rows": "loglevel.tsv", "key": "code", "name": "keyword" } diff --git a/data/misc/loglevel.tsv b/data/misc/loglevel.tsv index 8ba335c..922c626 100644 --- a/data/misc/loglevel.tsv +++ b/data/misc/loglevel.tsv @@ -1,9 +1,9 @@ -code severity -0 Emergency -1 Alert -2 Critical -3 Error -4 Warning -5 Notice -6 Informational -7 Debug +code keyword severity +0 emerg Emergency +1 alert Alert +2 crit Critical +3 err Error +4 warning Warning +5 notice Notice +6 info Informational +7 debug Debug -- 2.52.0 From e09e92ff030a891b4fb031ec505fdf5b1f35eb90 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 20 Sep 2026 21:21:10 +0200 Subject: [PATCH 08/10] Re-pin the shipped shape for misc.loglevel's keyword --- testdata/shipped_shape.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testdata/shipped_shape.txt b/testdata/shipped_shape.txt index b882ed4..b7cab31 100644 --- a/testdata/shipped_shape.txt +++ b/testdata/shipped_shape.txt @@ -240,8 +240,9 @@ misc.language format "{name}" key code name name misc.language.code string misc.language.code3 string misc.language.name string -misc.loglevel format "{severity}" key code name severity +misc.loglevel format "{keyword}" key code name keyword misc.loglevel.code string +misc.loglevel.keyword string misc.loglevel.severity string misc.mac format "{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}:{hex(2)}" misc.mimetype format "{type}" key type name ext -- 2.52.0 From ea9c1ad4369796a61ec9480331cf3814967a9c7f Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 20 Sep 2026 21:26:54 +0200 Subject: [PATCH 09/10] Name only the checker the keyword answers to, and file the Error near miss --- README.md | 2 +- data-import/loglevel.py | 8 ++++---- todo.md | 5 +++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e487bf6..e6b1108 100644 --- a/README.md +++ b/README.md @@ -1237,7 +1237,7 @@ the Development section below, and who ships a register the four above then draw - **`misc.loglevel` is a table keyed by the code, rendering POSIX's keyword.** A flat list of names carries neither the code a PRI encodes nor a selector reaching it, and goal 1 draws the two as one fact. RFC 5424 spells the severity `Informational`, - which no syslog, PSR-3 or `slog` checker accepts, so the shipped spelling is the one + which no syslog checker accepts, so the shipped spelling is the one a configuration writes, `info`, and the RFC's name stays the `severity` column. The code and the keyword take the two selector slots, so `misc.loglevel[Error]` misses. - **`misc.tld` keys carry the leading dot, where other tables key on a bare code.** diff --git a/data-import/loglevel.py b/data-import/loglevel.py index 67f2242..838ec15 100644 --- a/data-import/loglevel.py +++ b/data-import/loglevel.py @@ -5,8 +5,8 @@ Table 2 is the whole set, read between the facility table's caption and its own. The keyword is POSIX's severity macro less its `LOG_` prefix, lowercased, taken in the -order POSIX lists them, which `LOG_UPTO` states is the severity order; each must be a -prefix of the severity RFC 5424 numbers alike, so the two sources pin one pairing. +order POSIX lists them; each must be a prefix of the severity RFC 5424 numbers alike, +so the two sources pin one pairing. """ import argparse import html @@ -24,7 +24,7 @@ CACHE = Path(__file__).resolve().parent / "cache" COLUMNS = ["code", "keyword", "severity"] TABLE = re.compile(r"Table 1\.\s+Syslog Message Facilities(.*?)Table 2\.\s+Syslog Message Severities", re.S) ROW = re.compile(r"^ +(\d) +([A-Z][a-z]+): ", re.M) -SEVERITIES = re.compile(r"severity level portion of the.*?macros:(.*?)The following shall be declared as functions", re.S) +SECTION = re.compile(r"severity level portion of the.*?macros:(.*?)The following shall be declared as functions", re.S) MACRO = re.compile(r"\bLOG_([A-Z]+)\b") EXPECTED = 8 @@ -37,7 +37,7 @@ def severities(text): def keywords(page): - section = SEVERITIES.search(html.unescape(re.sub(r"<[^>]+>", "", page))) + section = SECTION.search(html.unescape(re.sub(r"<[^>]+>", "", page))) if not section: sys.exit("POSIX spells no severity list between its own caption and the function declarations; the page has moved") return [m.lower() for m in MACRO.findall(section.group(1))] diff --git a/todo.md b/todo.md index 6db496a..e314bb1 100644 --- a/todo.md +++ b/todo.md @@ -110,8 +110,9 @@ by a `data-import/` script, as README goal 10 asks. `misc.httpmethod[get]` and `misc.territory[se]` all answer "no row … has key or name" and stop there, where a case-insensitive match could name the row that exists, and a table with no `name` column could say it selects by key alone. `misc.tld[se]` misses - on the leading dot its keys carry rather than on case, so the near miss is worth - naming whatever shape it takes. + on the leading dot its keys carry rather than on case, and `misc.loglevel[Error]` + although the table carries `Error` in a column, so the near miss is worth naming + whatever shape it takes. - Let a table column carry a `datatype`, so `--format json` writes `"safe": true` and `--format sql` a boolean rather than the text `'true'`. Today only a JSON field takes one, so `misc.httpmethod`'s booleans are typed in Go and text -- 2.52.0 From ce59653ccd7f68eb563f15e95cc1db2bb39cef34 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 20 Sep 2026 21:29:55 +0200 Subject: [PATCH 10/10] Say what selects, name the decision the keyword instances, and cite POSIX's macros --- DATA-LICENSES.md | 2 +- README.md | 13 +++++++------ data-import/loglevel.py | 7 +++---- todo.md | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/DATA-LICENSES.md b/DATA-LICENSES.md index 48491a9..c59a400 100644 --- a/DATA-LICENSES.md +++ b/DATA-LICENSES.md @@ -19,7 +19,7 @@ Every shipped dataset, its source, its licence and the attribution it asks for. | `misc/httpmethod.tsv` | [IANA HTTP Method Registry](https://www.iana.org/assignments/http-methods/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpmethod.py` | | `misc/httpstatus.tsv` | [IANA HTTP Status Code Registry](https://www.iana.org/assignments/http-status-codes/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/httpstatus.py` | | `misc/language.tsv` | [datasets/language-codes](https://github.com/datasets/language-codes), the [Library of Congress](https://www.loc.gov/standards/iso639-2/) ISO 639-2 register | [PDDL 1.0](https://opendatacommons.org/licenses/pddl/1-0/) | none required | `data-import/language.py` | -| `misc/loglevel.tsv` | [RFC 5424](https://www.rfc-editor.org/rfc/rfc5424.txt) Table 2, the syslog message severities; keywords from [POSIX](https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/syslog.h.html) `` | facts | none required | `data-import/loglevel.py` | +| `misc/loglevel.tsv` | [RFC 5424](https://www.rfc-editor.org/rfc/rfc5424.txt) Table 2, the syslog message severities; keywords from the [POSIX](https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/syslog.h.html) `` severity macros | facts | none required | `data-import/loglevel.py` | | `misc/mimetype.tsv` | [mime-db](https://github.com/jshttp/mime-db), the IANA media type registry with filename extensions | [MIT](https://github.com/jshttp/mime-db/blob/master/LICENSE) | "Copyright (c) 2014 Jonathan Ong, Copyright (c) 2015-2022 Douglas Christopher Wilson" | `data-import/mimetype.py` | | `misc/port.tsv` | [IANA Service Name and Transport Protocol Port Number Registry](https://www.iana.org/assignments/service-names-port-numbers/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/port.py` | | `misc/protocol.tsv` | [IANA Protocol Numbers](https://www.iana.org/assignments/protocol-numbers/) | [public domain](https://www.iana.org/help/licensing-terms) | none required | `data-import/protocol.py` | diff --git a/README.md b/README.md index e6b1108..d65badf 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,8 @@ a fixture needs one a reader recognises. `misc.loglevel` is the eight syslog severities, rendering the keyword a configuration writes, `err` and `info`, keyed by the numerical code a PRI encodes and selectable by either, so `misc.loglevel[3]` and `misc.loglevel[err]` are one row. -`misc.loglevel.severity` is the name RFC 5424 spells, `Error` beside `err`. +`misc.loglevel.severity` is the name RFC 5424 spells, `Error` beside `err`; it +selects nothing, the code and the keyword do. [`DATA-LICENSES.md`](DATA-LICENSES.md) names each table's source and licence. `misc.timezone` is every zone tzdb gives a shipped territory, from one apiece for most @@ -761,7 +762,7 @@ a minor only adds, and a major is the only release that changes what exists. | Surface | Major | Minor | |---------|-------|-------| | Shipped data | remove or rename a path; change a category's format; remove a value, or change a weight or a repeat; add a reference from one shipped category into another; change a table's key, name, weight or parent column, or remove a row | a path outside a record's columns, a locale, a value in a list, a row | -| Records | remove, rename, retype or add a column; let a column be null | a record, as a new category | +| Records (a table is one) | remove, rename, retype or add a column; let a column be null | a record, as a new category | | Data format | a fence: a spelling `New` rejects that it accepted; a template option, since it reserves a field name | a builtin | | CLI | remove or rename a flag, or change its default; change what an exit code means; change the framing a `--format` writes (header, quoting, statement shape), the `--list` layout, or what an error names | a flag, a format | | Library | change or remove an exported name; raise the lowest supported Go | an exported name, a `With…` option | @@ -1236,10 +1237,10 @@ the Development section below, and who ships a register the four above then draw fejkdata has no business making. - **`misc.loglevel` is a table keyed by the code, rendering POSIX's keyword.** A flat list of names carries neither the code a PRI encodes nor a selector reaching it, and - goal 1 draws the two as one fact. RFC 5424 spells the severity `Informational`, - which no syslog checker accepts, so the shipped spelling is the one - a configuration writes, `info`, and the RFC's name stays the `severity` column. The - code and the keyword take the two selector slots, so `misc.loglevel[Error]` misses. + goal 1 draws the two as one fact. The canonical spelling losing to the one its domain + writes, above, settles the rest: a configuration writes `info`, so RFC 5424's + `Informational` stays the `severity` column, and the code and the keyword hold the + two selector slots. - **`misc.tld` keys carry the leading dot, where other tables key on a bare code.** The register spells a TLD `.se` and `misc.territory.tld` already ships it so, which a bare key would make two spellings of one fact; `{/misc.tld}` also composes onto a diff --git a/data-import/loglevel.py b/data-import/loglevel.py index 838ec15..d9da6d5 100644 --- a/data-import/loglevel.py +++ b/data-import/loglevel.py @@ -3,10 +3,9 @@ data-import/loglevel.py [--source URL_OR_FILE] [--posix URL_OR_FILE] [--cache DIR] [--out FILE] -Table 2 is the whole set, read between the facility table's caption and its own. The -keyword is POSIX's severity macro less its `LOG_` prefix, lowercased, taken in the -order POSIX lists them; each must be a prefix of the severity RFC 5424 numbers alike, -so the two sources pin one pairing. +Every severity in Table 2 ships, paired to POSIX's keywords by the order each source +lists them, which holds only because a keyword prefixes the severity RFC 5424 numbers +alike. """ import argparse import html diff --git a/todo.md b/todo.md index e314bb1..c8d1311 100644 --- a/todo.md +++ b/todo.md @@ -111,8 +111,8 @@ by a `data-import/` script, as README goal 10 asks. and stop there, where a case-insensitive match could name the row that exists, and a table with no `name` column could say it selects by key alone. `misc.tld[se]` misses on the leading dot its keys carry rather than on case, and `misc.loglevel[Error]` - although the table carries `Error` in a column, so the near miss is worth naming - whatever shape it takes. + misses although a column carries `Error`, so the near miss is worth naming whatever + shape it takes. - Let a table column carry a `datatype`, so `--format json` writes `"safe": true` and `--format sql` a boolean rather than the text `'true'`. Today only a JSON field takes one, so `misc.httpmethod`'s booleans are typed in Go and text -- 2.52.0