Refuse a blank TSV line, fence a table's family through any template, resolve a path before drawing, clamp the weighted pick, and say which countries and currencies the tables hold

CHANGELOG.md

@@ -21,7 +21,8 @@ replacement, and each removed path, column or flag.
 - `New` refuses a root choice of templates sharing one format and one set of string
   fields, naming the rows TSV to write instead.
 - `misc.country`, `misc.currency`, `misc.language`, `misc.httpstatus` and
-  `misc.mimetype` are tables. `misc.country` is the full ISO 3166 register with the
-  columns `calling-code`, `capital`, `currency`, `flag`, `languages`, `numeric` and
-  `tld` added; `misc.currency` the current ISO 4217 list with `decimals` and
-  `numeric` added, and its symbols from CLDR. `DATA-LICENSES.md` lists each source.
+  `misc.mimetype` are tables. `misc.country` is every ISO 3166 country that has a
+  capital, a currency and a TLD, with the columns `calling-code`, `capital`,
+  `currency`, `flag`, `languages`, `numeric` and `tld` added; `misc.currency` the
+  current ISO 4217 currencies with a minor unit, with `decimals` and `numeric`
+  added, and its symbols from CLDR. `DATA-LICENSES.md` lists each source.

DATA-LICENSES.md

@@ -12,7 +12,3 @@ Every shipped dataset, its source, its licence and the attribution it asks for.
 | `misc/mimetype.tsv` | curated (IANA media types are facts) | — | — | — |
 
 Every other category is hand-written JSON under [`data/`](data), MIT like the code.
-
-```sh
-docker compose run --rm --user "$(id -u):$(id -g)" data-import data-import/country.py
-```

compose.yaml

@@ -64,7 +64,6 @@ services:
     <<: *go
     command: sh
 
-  # Rebuilds a shipped TSV from its source: docker compose run --rm data-import data-import/country.py
   data-import:
     image: python:3.14.7-slim
     working_dir: /app

draw.go

@@ -144,12 +144,34 @@ func (c *drawCheck) checkDraws(path string, n node) error {
 	for _, e := range renderEdges(t) {
 		w.edge(t, e, drawAt{group: t.drawGroupKey, route: drawRoute{e.reached(), e.label}})
 	}
+	if err := w.checkOwnFamily(t); err != nil {
+		return fmt.Errorf("%s: %w", path, err)
+	}
 	if err := w.check(); err != nil {
 		return fmt.Errorf("%s: %w", path, err)
 	}
 	return nil
 }
 
+// checkOwnFamily refuses a table's format or cell that reads, however many templates
+// away, a table of its own family: a row rendered whole draws its row without
+// pinning it, so the family would draw apart from the row being rendered.
+func (w *drawWalk) checkOwnFamily(t *template) error {
+	own := t.table
+	if own == nil {
+		own = t.cellOf
+	}
+	if own == nil {
+		return nil
+	}
+	for _, r := range w.reads {
+		if r.tr != nil && r.tr.head.family() == own.family() {
+			return fmt.Errorf("%s reads %s, a table of its own family, which a row of %s rendered whole would draw apart from; read the family from a template beside it, or add the value as a column", r.at.route.spelled(r.a.name), r.tr.head.category, own.category)
+		}
+	}
+	return nil
+}
+
 // checkRecordDraws fences the columns of a record — a template compiled at the top without a
 // repeat — so a load proves the record view of it as well as the string view.
 func (c *drawCheck) checkRecordDraws(path string, n node) error {

node.go

@@ -65,6 +65,7 @@ type template struct {
 	readsColumn  *columnRead // set when the format is one reference alone reading a record's column
 	record       bool        // compiled at the top without a repeat, so its fields are record columns
 	table        *table      // the table whose format this is, whose columns are the fields
+	cellOf       *table      // the table whose cell this is
 	drawGroup    string      // the draw group it draws in, as written; "" keeps its caller's
 	drawGroupKey string      // its draw group keyed by its category once linked: what a render reads its reference paths under
 }

path.go

@@ -141,8 +141,7 @@ type pathWalk struct {
 // walkPath descends tail from n and returns the node it ends at: a folder or
 // template by its next segment, a choice by w.choice, which consumes no segment, a
 // table by walkTable. A missing segment is an error, so no walk reaches past what
-// the data holds. A table-shaped dispatch, one case per node kind, kept whole on
-// purpose.
+// the data holds.
 func walkPath(n node, tail []string, w pathWalk) (node, error) {
 	if len(tail) == 0 {
 		return n, w.atLeaf(n)
@@ -216,33 +215,57 @@ func walkChoice(c *choice, tail []string, w pathWalk) (node, error) {
 // table was reached from another; a table reached whole, with no selector, is left
 // to a render's own draw.
 func walkTable(t *table, tail []string, w pathWalk, descended bool) (node, error) {
-	sel := ""
-	if len(tail) > 0 && isSelector(tail[0]) {
-		sel, tail = selectorOf(tail[0]), tail[1:]
-		if len(tail) > 0 && isSelector(tail[0]) {
-			return nil, fmt.Errorf("%s[%s] is selected twice; one selector names its row", t.category, sel)
+	sel, tail, err := t.selector(tail)
+	if err != nil {
+		return nil, err
 	}
+	column, child, err := t.step(tail)
+	if err != nil {
+		return nil, err
 	}
-	// A selector further down pins this table by ancestry, so the walk draws only
+	// Resolved before any draw, so a path that fails moves no seeded stream. A
+	// selector further down pins this table by ancestry, so the walk draws only
 	// where none follows; drawing first could pick a row the selector is not inside.
 	if w.pins != nil {
 		if err := readRow(w.pins, t, sel, (descended || len(tail) > 0) && !hasSelector(tail)); err != nil {
 			return nil, err
 		}
 	}
-	if len(tail) == 0 {
-		if sel == "" && !descended {
+	switch {
+	case len(tail) == 0 && sel == "" && !descended:
 		return walkPath(t, nil, w)
-		}
+	case len(tail) == 0:
 		return walkPath(t.whole, nil, w)
-	}
-	if i, ok := t.col[tail[0]]; ok {
-		return walkPath(t.fields[t.columns[i]], tail[1:], w)
-	}
-	if child := t.descendant(tail[0]); child != nil {
+	case child != nil:
 		return walkTable(child, tail[1:], w, true)
 	}
-	return nil, fmt.Errorf("no column or linked table %q in %s", tail[0], t.category)
+	return walkPath(column, tail[1:], w)
+}
+
+// selector splits the selector a tail starts with from the rest of it.
+func (t *table) selector(tail []string) (sel string, rest []string, err error) {
+	if len(tail) == 0 || !isSelector(tail[0]) {
+		return "", tail, nil
+	}
+	sel, rest = selectorOf(tail[0]), tail[1:]
+	if len(rest) > 0 && isSelector(rest[0]) {
+		return "", nil, fmt.Errorf("%s[%s] is selected twice; one selector names its row", t.category, sel)
+	}
+	return sel, rest, nil
+}
+
+// step is what a tail's first segment names in t: a column, or a table linked to it.
+func (t *table) step(tail []string) (column node, child *table, err error) {
+	if len(tail) == 0 {
+		return nil, nil, nil
+	}
+	if i, ok := t.col[tail[0]]; ok {
+		return t.fields[t.columns[i]], nil, nil
+	}
+	if child = t.descendant(tail[0]); child == nil {
+		return nil, nil, fmt.Errorf("no column or linked table %q in %s", tail[0], t.category)
+	}
+	return nil, child, nil
 }
 
 // readRow pins the row a path reads of t: the one its selector names, or, where the

reference.go

@@ -109,9 +109,6 @@ func linkTemplateRefs(folder []string, path, category string, t *template, root
 		if category != "" && key == "/"+category {
 			return fmt.Errorf("%s: reference {%s}: names the category it sits in; read a sibling field as a path, or move the shared value into its own category and reference that", path, name)
 		}
-		if err := checkNotOwnFamily(root, category, target); err != nil {
-			return fmt.Errorf("%s: reference {%s}: %w", path, name, err)
-		}
 		if err := checkPath(target, tail, key); err != nil {
 			return fmt.Errorf("%s: reference {%s}: %w", path, name, err)
 		}
@@ -125,22 +122,6 @@ func linkTemplateRefs(folder []string, path, category string, t *template, root
 	return nil
 }
 
-// checkNotOwnFamily refuses a reference from a table's format or cell into a table of
-// its own family: a table rendered whole draws its row without pinning it, so the
-// family would draw apart from the row being rendered.
-func checkNotOwnFamily(root map[string]node, category string, target node) error {
-	if category == "" {
-		return nil
-	}
-	_, own, _, err := resolveCategory(root, strings.Split(category, "."))
-	from, isTable := own.(*table)
-	into, targetIsTable := target.(*table)
-	if err != nil || !isTable || !targetIsTable || from.family() != into.family() {
-		return nil
-	}
-	return fmt.Errorf("a cell or format of %s reads %s, a table of its own family, which a row of %s rendered whole would draw apart from; read the family from a template beside it, or add the value as a column", from.category, into.category, from.category)
-}
-
 // columnRead is a record's column read by a format of that one reference alone, which is the
 // column: it takes the column's datatype and null.
 type columnRead struct {

table.go

@@ -171,21 +171,36 @@ func (t *table) parseRows(data string) error {
 	}
 	m := len(t.columns)
 	t.cells = make([]string, 0, m*(strings.Count(rest, "\n")+1))
-	for line, n := 2, 0; rest != ""; line++ {
-		var row string
-		row, rest, _ = strings.Cut(rest, "\n")
+	for line := 2; ; line++ {
+		row, more, found := strings.Cut(rest, "\n")
 		row = strings.TrimSuffix(row, "\r")
-		for n = 0; n < m; n++ {
-			cell, more, tab := strings.Cut(row, "\t")
+		if err := t.appendRow(row, line); err != nil {
+			return err
+		}
+		if !found {
+			break
+		}
+		rest = more
+	}
+	if t.rows() < 2 {
+		return fmt.Errorf("has one row, which is a template; write it as one")
+	}
+	return nil
+}
+
+// appendRow splits one line into as many cells as the header has columns.
+func (t *table) appendRow(row string, line int) error {
+	if row == "" {
+		return fmt.Errorf("line %d is empty; every line below the header is a row", line)
+	}
+	m := len(t.columns)
+	for n := 0; n < m; n++ {
+		cell, next, tab := strings.Cut(row, "\t")
 		if !tab && n < m-1 || tab && n == m-1 {
 			return fmt.Errorf("line %d has %s cells than the %d columns", line, map[bool]string{true: "more", false: "fewer"}[tab], m)
 		}
 		t.cells = append(t.cells, cell)
-			row = more
-		}
-	}
-	if t.rows() == 1 {
-		return fmt.Errorf("has one row, which is a template; write it as one")
+		row = next
 	}
 	return nil
 }
@@ -280,6 +295,7 @@ func (t *table) checkCells() error {
 			t.tokens = map[int]*template{}
 		}
 		t.tokens[i] = n.(*template)
+		t.tokens[i].cellOf = t
 	}
 	return nil
 }
@@ -395,7 +411,8 @@ func (t *table) indexed() *tableIndex {
 				for k, rows := range t.index.children {
 					cum, total := make([]float64, len(rows)), 0.0
 					for i, r := range rows {
-						total += t.cum[r] - t.cumBefore(r)
+						w, _ := strconv.ParseFloat(t.cell(r, t.weight), 64) // sumWeights proved it
+						total += w
 						cum[i] = total
 					}
 					t.index.childCum[k] = cum
@@ -406,13 +423,6 @@ func (t *table) indexed() *tableIndex {
 	return &t.index
 }
 
-func (t *table) cumBefore(r int) float64 {
-	if r == 0 {
-		return 0
-	}
-	return t.cum[r-1]
-}
-
 // draw picks a row over the whole table. The session is concrete rather than the rng
 // interface so that a walk holding the draws allocates nothing.
 func (t *table) draw(s *session) int {
@@ -424,7 +434,7 @@ func (t *table) draw(s *session) int {
 
 func pickCum(s *session, cum []float64) int {
 	x := s.Float64() * cum[len(cum)-1]
-	return sort.Search(len(cum), func(i int) bool { return cum[i] > x })
+	return min(sort.Search(len(cum), func(i int) bool { return cum[i] > x }), len(cum)-1) // x can round up to the total
 }
 
 // drawUnder picks a row among those linked to parent row pr.