Keep a read's null with its draw, share the lone-reference check, name a JSON-string item's object form again, prove each column once per pass, and read a record off template.record

README.md

@@ -310,9 +310,9 @@ order.id: datatype integer: {digits(3)} prints text, not an integer
 order.id: datatype integer: "1{digits(2)}" is not one value; write one literal or one {int()}, {float()}, {seq()} or {calc()}, or read one
 ```
 
-A column of one reference alone to another record's column — `"score": "{/src.score}"`,
+A column of one reference alone to another record's column — `"score": "{/src.score}"`
-or a struct field tagged `src.score` — is that column: it takes the column's datatype
+— is that column: it takes the column's datatype and is null where the column is, and
-and is null, or nil, where the column is. A `datatype` of its own types a string
+a struct field tagged `src.score` is nil there. A `datatype` of its own types a string
 column's values, and over a typed column is refused naming the datatype it takes. Any
 other read renders the column's text, a null as `""`.
 
@@ -336,7 +336,7 @@ renders a null as `""`. The other items' weights skew its odds:
 ```
 
 `deleted_at` is null every draw, `middle` a name three draws in four. Rejected at
-load: `null` anywhere but a column, naming `""`, and a column whose items declare
+load: `null` anywhere but a column, naming `""`, and a column whose items hold
 different datatypes.
 
 ### Options and fields

datatype.go

@@ -94,7 +94,7 @@ func columnDatatype(n node) (DataType, error) {
 	return first, nil
 }
 
-// itemDatatype is the datatype a column item declares, else that of the column it reads whole.
+// itemDatatype is the datatype a column item declares, else that of the column it is.
 func itemDatatype(t *template) DataType {
 	if t.datatype != DataTypeString {
 		return t.datatype
@@ -102,17 +102,20 @@ func itemDatatype(t *template) DataType {
 	return readDatatype(t)
 }
 
-// readDatatype is the datatype of the column t reads whole; a string when it reads none.
+// readDatatype is the datatype of the column t is, reading it by one reference alone; a string
+// when t reads none.
 func readDatatype(t *template) DataType {
-	if t.inherits == nil {
+	if t.readsColumn == nil {
 		return DataTypeString
 	}
-	d, _ := columnDatatype(t.inherits) // checkColumns refuses that column where it sits
+	items, _ := columnItems(t.readsColumn.column)
-	return d
+	if len(items) == 0 {
+		return DataTypeString
+	}
+	return itemDatatype(items[0]) // checkColumns refuses that column where its items disagree
 }
 
-// columnItems is a column's template items, its choices unwrapped, and whether one is null
+// columnItems is a column's template items, its choices unwrapped, and whether one is null.
-// or reads whole a column that can be.
 func columnItems(n node) (items []*template, nullable bool) {
 	var collect func(node)
 	collect = func(n node) {
@@ -123,10 +126,6 @@ func columnItems(n node) (items []*template, nullable bool) {
 			}
 		case *template:
 			items = append(items, n)
-			if n.inherits != nil {
-				_, inherited := columnItems(n.inherits)
-				nullable = nullable || inherited
-			}
 		case *null:
 			nullable = true
 		}
@@ -143,8 +142,8 @@ func disagreement(a *template, da DataType, b *template, db DataType) error {
 	}
 	switch {
 	case da != DataTypeString && db != DataTypeString:
-		return fmt.Errorf("its items declare %s and %s; a column holds one datatype", da, db)
+		return fmt.Errorf("its items hold %s and %s; a column holds one datatype", da, db)
-	case bare.fields == nil: // a JSON string; an object, which may carry a weight, has a fields map
+	case bare.fromString: // an object may carry a weight, which this spelling would drop
 		return fmt.Errorf(`item %q declares no datatype, and a column holds one; write it as {"format":%q,"datatype":%q}`, bare.format, bare.format, want)
 	}
 	return fmt.Errorf(`item %q declares no datatype beside one declaring %s; a column holds one, so give it "datatype": %q`, bare.format, want, want)

hold.go

@@ -253,20 +253,17 @@ func checkNoRepeatedRead(format string, c formatOps, refs map[string]refBinding)
 }
 
 // draws is what an expansion has already drawn for its held names: the variant each
-// was drawn as, so every path under it reads one row, the value each read, by its one
+// was drawn as, so every path under it reads one row, and the draw each read made, by
-// spelling, so the same read written twice reads one value, and the columns drawn null,
+// its one spelling, so the same read written twice reads one draw.
-// so a read of one whole is null too.
 type draws struct {
 	variant map[string]node
-	value   map[string]string
+	value   map[string]draw
-	nulls   map[node]bool
 }
 
-func (d *draws) drewNull(column node) {
+// draw is what one read drew: its text, and whether it landed on a null.
-	if d.nulls == nil {
+type draw struct {
-		d.nulls = map[node]bool{}
+	text string
-	}
+	null bool
-	d.nulls[column] = true
 }
 
 // readField renders one arm of a token. An arm's key is a sibling field or a
@@ -276,23 +273,18 @@ func (d *draws) drewNull(column node) {
 // gives one value, and a shown operand is the operand computed. Every other name is
 // drawn afresh, so {word} {word} still draws twice. checkTokens, checkPath and
 // linkRefs prove every step, so the walk cannot fail.
-func readField(s *session, t *template, held, refScope *draws, a arm) string {
+func readField(s *session, t *template, held, refScope *draws, a arm) draw {
 	if !t.held[a.key] {
 		if len(a.tail) > 0 {
 			panic(fmt.Sprintf("fejkdata: %q reads a path into %q, which the expansion does not hold", a.name, a.key))
 		}
-		return render(s, t.fields[a.key], refScope)
+		return draw{text: render(s, t.fields[a.key], refScope)}
 	}
-	// A reference that reads a path reads the caller's scope, so its draw outlives
+	d := readScope(held, refScope, a)
-	// this expansion; a sibling, and a reference read whole, stay local to it.
+	if r, done := d.value[a.path]; done {
-	d := held
+		return r
-	if isRef(a.key) && refScope != nil && len(a.tail) > 0 {
-		d = refScope
 	}
-	if v, read := d.value[a.path]; read {
+	var r draw
-		return v
-	}
-	var v string
 	_ = walkPath(t.fields[a.key], a.tail, pathWalk{
 		// Hold the draw at every level passed through, so two paths sharing a
 		// prefix share it.
@@ -308,10 +300,35 @@ func readField(s *session, t *template, held, refScope *draws, a arm) string {
 			}
 			return []node{n}, nil
 		},
-		leaf: func(n node) error { v, _ = renderColumn(s, n, refScope); return nil },
+		leaf: func(n node) error { r = renderLeaf(s, n, refScope); return nil },
 	})
-	d.value[a.path] = v
+	d.value[a.path] = r
-	return v
+	return r
+}
+
+// readScope is the draws a held read keeps its draw in: for a reference that reads a path,
+// refScope, so its draw outlives this expansion; for a sibling, or a reference read whole, held.
+func readScope(held, refScope *draws, a arm) *draws {
+	if isRef(a.key) && refScope != nil && len(a.tail) > 0 {
+		return refScope
+	}
+	return held
+}
+
+// renderLeaf draws and renders what a read lands on: null on a null item, or on a column of one
+// reference alone whose read drew null.
+func renderLeaf(s *session, n node, scope *draws) draw {
+	n = drawn(s, n)
+	if _, isNull := n.(*null); isNull {
+		return draw{null: true}
+	}
+	r := draw{text: render(s, n, scope)}
+	if t, _ := n.(*template); t != nil && t.readsColumn != nil {
+		if d := readScope(nil, scope, t.readsColumn.a); d != nil {
+			r.null = d.value[t.readsColumn.a.path].null
+		}
+	}
+	return r
 }
 
 // drawn resolves a choice to one variant, so a bound head is a concrete node the

inline.go

@@ -101,12 +101,8 @@ func loneReference(arg string) (string, bool) {
 		raw = m["format"]
 	}
 	format, isString := raw.(string)
-	var units []ftoken
+	body, lone := loneRef(format)
-	if !isString || eachToken(format, func(t ftoken) error { units = append(units, t); return nil }) != nil || len(units) != 1 {
+	if !isString || !lone {
-		return "", false
-	}
-	body := units[0].body
-	if units[0].kind != 'b' || !isRef(body) || strings.ContainsAny(body, "|(") {
 		return "", false
 	}
 	if strings.HasPrefix(body, "/") {

node.go

@@ -59,8 +59,8 @@ type template struct {
 	// held is every name drawn once per expansion: the bound levels above, plus the
 	// siblings a {calc()} reads. nil when the format holds nothing (see expand).
 	held        map[string]bool
-	// inherits is the column a format of one reference alone reads, taking its datatype and null.
+	fromString  bool        // written as a JSON string rather than an object
-	inherits node
+	readsColumn *columnRead // set when the format is one reference alone reading a record's column
 	record      bool        // compiled at the top without a repeat, so its fields are record columns
 }
 
@@ -127,7 +127,7 @@ func compileString(s string) (node, error) {
 	if err := checkTokens(s, nil); err != nil {
 		return nil, err
 	}
-	t := &template{format: s, repeat: 1}
+	t := &template{format: s, repeat: 1, fromString: true}
 	if err := t.compileFormat(); err != nil {
 		return nil, err
 	}
@@ -234,7 +234,7 @@ func compileTemplate(m map[string]any, pos position) (node, error) {
 		return nil, err
 	}
 	fieldPos := inFormat
-	if pos == atTop && projectsColumns(o.repeat) {
+	if pos == atTop && o.repeat == 1 {
 		fieldPos = inColumn
 	}
 	fields, err := compileFields(m, fieldPos)

record.go

@@ -205,13 +205,13 @@ func recordOf(n node) (*template, []Column, error) {
 	if !ok {
 		return nil, nil, errors.New("names a choice, not a template; a record is a template whose fields are its columns")
 	}
-	if !projectsColumns(t.repeat) {
-		return nil, nil, fmt.Errorf("carries repeat %d, which composes its format into one string; a record projects columns instead — drop the repeat and render the record again for more rows", t.repeat)
-	}
 	names := recordColumns(t)
 	if len(names) == 0 {
 		return nil, nil, errors.New("has no fields, so no columns")
 	}
+	if !t.record {
+		return nil, nil, fmt.Errorf("carries repeat %d, which composes its format into one string; a record projects columns instead — drop the repeat and render the record again for more rows", t.repeat)
+	}
 	if err := checkColumnRefs(t, names); err != nil {
 		return nil, nil, err
 	}
@@ -223,10 +223,6 @@ func recordOf(n node) (*template, []Column, error) {
 	return t, columns, nil
 }
 
-// projectsColumns reports whether a category or inline template with this repeat is a
-// record, its fields the columns; a repeat composes the format into one string instead.
-func projectsColumns(repeat int) bool { return repeat == 1 }
-
 // checkColumnRefs rejects the reference reads a record's shared draw cannot answer
 // for: one column rendering a level another reads a path into, and a column
 // reading the record back through its own path.
@@ -297,30 +293,15 @@ func columnRefs(t *template, columns []string) ([]columnRef, error) {
 // renderRecord draws each column once, in the name order recordOf fixed, over one
 // reference scope shared across them.
 func renderRecord(s *session, t *template, columns []Column) *Record {
-	scope := &draws{variant: map[string]node{}, value: map[string]string{}}
+	scope := &draws{variant: map[string]node{}, value: map[string]draw{}}
 	r := &Record{columns: append([]Column(nil), columns...)}
 	for i := range r.columns {
-		r.columns[i].Value, r.columns[i].Null = renderColumn(s, t.fields[r.columns[i].Name], scope)
+		column := renderLeaf(s, t.fields[r.columns[i].Name], scope)
+		r.columns[i].Value, r.columns[i].Null = column.text, column.null
 	}
 	return r
 }
 
-// renderColumn draws a column and reports whether the draw is null: a null item, or an item
-// reading whole a column that scope drew null.
-func renderColumn(s *session, column node, scope *draws) (string, bool) {
-	n := drawn(s, column)
-	value, isNull := "", true
-	if _, drewNull := n.(*null); !drewNull {
-		value = render(s, n, scope)
-		t, _ := n.(*template)
-		isNull = t != nil && t.inherits != nil && scope != nil && scope.nulls[t.inherits]
-	}
-	if isNull && scope != nil {
-		scope.drewNull(column)
-	}
-	return value, isNull
-}
-
 // recordColumns is the sorted non-reference field names — the columns a record
 // projects. A {/path} binding is carried in fields under its root path, so only a
 // name that is not a reference is a column.

reference.go

@@ -107,22 +107,42 @@ func linkTemplateRefs(folder []string, path string, t *template, root map[string
 	if err := t.compileFormat(); err != nil {
 		return fmt.Errorf("%s: %w", path, err)
 	}
-	t.inherits = columnRead(t)
+	t.readsColumn = columnReadOf(t)
 	return nil
 }
 
-// columnRead is the column t reads whole: its format is one reference alone, reading a field of
+// columnRead is a record's column read by a format of that one reference alone, which is the
-// a record, so what that column draws is what t draws.
+// column: it takes the column's datatype and null.
-func columnRead(t *template) node {
+type columnRead struct {
-	if t.repeat != 1 || len(t.ops) != 1 || t.ops[0].kind != 'f' || len(t.ops[0].arms) != 1 {
+	a      arm
+	column node
+}
+
+func columnReadOf(t *template) *columnRead {
+	name, lone := loneRef(t.format)
+	if !lone || t.repeat != 1 {
 		return nil
 	}
-	a := t.ops[0].arms[0]
+	a := splitArm(name, t.refs)
 	target, isTemplate := t.fields[a.key].(*template)
-	if !isRef(a.name) || !isTemplate || !target.record || len(a.tail) != 1 {
+	if !isTemplate || !target.record || len(a.tail) != 1 {
 		return nil
 	}
-	return target.fields[a.tail[0]]
+	return &columnRead{a: a, column: target.fields[a.tail[0]]}
+}
+
+// loneRef is the reference a format of one reference token and nothing else reads.
+func loneRef(format string) (string, bool) {
+	units, body := 0, ""
+	err := eachToken(format, func(t ftoken) error {
+		units++
+		body = t.body
+		return nil
+	})
+	if err != nil || units != 1 || !isRef(body) || strings.ContainsAny(body, "|(") {
+		return "", false
+	}
+	return body, true
 }
 
 // eachTemplate calls fn once per template, with the folder its category sits in

render.go

@@ -103,7 +103,7 @@ func expand(s *session, t *template, refScope *draws) string {
 	if len(t.held) > 0 {
 		held = &draws{
 			variant: make(map[string]node, len(t.held)),
-			value:   make(map[string]string, len(t.held)),
+			value:   make(map[string]draw, len(t.held)),
 		}
 	}
 	for i := range t.ops {
@@ -112,7 +112,7 @@ func expand(s *session, t *template, refScope *draws) string {
 		case 'l':
 			b.WriteString(o.lit)
 		case 'f':
-			b.WriteString(readField(s, t, held, refScope, o.arms[s.IntN(len(o.arms))]))
+			b.WriteString(readField(s, t, held, refScope, o.arms[s.IntN(len(o.arms))]).text)
 		case 'b':
 			// Read before the call, so the value a calc computes is the value the
 			// format showed. calcVars fixed the order op.operands holds.
@@ -120,7 +120,7 @@ func expand(s *session, t *template, refScope *draws) string {
 			if len(o.operands) > 0 {
 				operands = make([]string, len(o.operands))
 				for j, a := range o.operands {
-					operands[j] = readField(s, t, held, refScope, a)
+					operands[j] = readField(s, t, held, refScope, a).text
 				}
 			}
 			b.WriteString(o.call(s, b.String(), operands)) // b.String() is the output so far

struct.go

@@ -318,7 +318,7 @@ func (k columnKind) holds(v proven) bool {
 // checkField rejects a column a field of Go type ft cannot fill: a datatype, which the Go type
 // sets, a null outside a pointer, or a value its kind's datatype or range refuses.
 func (p *valueProof) checkField(label string, ft reflect.Type, column node) error {
-	items, nullable := columnItems(column)
+	items, _ := columnItems(column)
 	for _, it := range items {
 		if it.datatype != DataTypeString {
 			return fmt.Errorf("%s: its Go type %s sets the datatype; drop \"datatype\"", label, ft)
@@ -327,7 +327,7 @@ func (p *valueProof) checkField(label string, ft reflect.Type, column node) erro
 	elem := ft
 	if ft.Kind() == reflect.Pointer {
 		elem = ft.Elem()
-	} else if nullable {
+	} else if p.column(column).null {
 		return fmt.Errorf("%s: its tag can draw null, which %s cannot hold; make it *%s", label, ft, ft)
 	}
 	kind := columnKinds[elem.Kind()]

todo.md

@@ -24,7 +24,9 @@ The record API lands first, so the data update can use it.
   blocks as columns (`sv_SE.person` → `femalefirst`, `malefirst`; `misc.uuid` →
   `variant`), and `sv_SE.address` draws its postal code apart from its locality.
   It also settles what a version promises about shipped data: its paths, its
-  record columns, and whether a seed renders the same output across versions.
+  record columns with their datatypes and nulls — a column of one reference alone
+  takes both, so typing a column or adding a null breaks its readers — and whether
+  a seed renders the same output across versions.
 - `email.local` and `username` share their handle lists, while their name
   variants differ on purpose. Share the lists only if that is a clean win.
 

value.go

@@ -16,15 +16,17 @@ type proven struct {
 	integral   bool
 	notOperand string // why some render reads as no finite number, the way calc reads it
 	not        [len(dataTypeNames)]string
+	null       bool // some draw of a column is null
 }
 
 // valueProof proves what typed columns and their calc operands hold, each node once per scope.
 type valueProof struct {
 	memo    map[node]proven
+	columns map[node]proven
 }
 
 // checkDatatype rejects a typed column item some render of which is not text of its datatype,
-// and one declaring a datatype over the typed column it reads whole.
+// and one declaring a datatype over the typed column it is.
 func (p *valueProof) checkDatatype(path string, n node) error {
 	t, ok := n.(*template)
 	if !ok || t.datatype == DataTypeString {
@@ -39,14 +41,25 @@ func (p *valueProof) checkDatatype(path string, n node) error {
 	return nil
 }
 
-// columnItem proves a column item: what it renders, or, reading a column whole, that column's
+// columnItem proves a column item: what it renders, or, when it is a column it reads, that column.
-// items, whose nulls it draws as null rather than rendering them.
 func (p *valueProof) columnItem(t *template) proven {
-	if t.inherits == nil {
+	if t.readsColumn == nil {
 		return p.of(t)
 	}
+	return p.column(t.readsColumn.column)
+}
+
+// column proves a column over what its items draw, a null item marking it null rather than
+// rendering "".
+func (p *valueProof) column(n node) proven {
+	if v, done := p.columns[n]; done {
+		return v
+	}
+	if p.columns == nil {
+		p.columns = map[node]proven{}
+	}
+	items, nullable := columnItems(n)
 	var v proven
-	items, _ := columnItems(t.inherits)
 	for i, it := range items {
 		if w := p.columnItem(it); i == 0 {
 			v = w
@@ -54,6 +67,8 @@ func (p *valueProof) columnItem(t *template) proven {
 			v = v.or(w)
 		}
 	}
+	v.null = v.null || nullable
+	p.columns[n] = v
 	return v
 }
 
@@ -88,7 +103,7 @@ func (p *valueProof) unite(nodes []node) proven {
 // or is what a proof knows of a render that is either v or w.
 func (v proven) or(w proven) proven {
 	v.lo, v.hi, v.nonZero = min(v.lo, w.lo), max(v.hi, w.hi), min(v.nonZero, w.nonZero)
-	v.integral = v.integral && w.integral
+	v.integral, v.null = v.integral && w.integral, v.null || w.null
 	if v.notOperand == "" {
 		v.notOperand = w.notOperand
 	}