Load a shipped category on the first call reaching it, from an index generated from data/

AGENTS.md

@@ -43,6 +43,7 @@ In [docs/decisions.md](docs/decisions.md):
 - A padded JSON argument is rejected, not trimmed
 - `FakeTemplate` and `NewTemplate` both stay
 - The shipped data is embedded, not discovered
+- The shipped set alone loads a category on the first call reaching it; beside a `--data-path` it loads whole
 - A bare reference draws each time; a reference path is held
 - Reference sigils follow the filesystem
 - A change to what exists is a major; a minor only adds

README.md

@@ -813,9 +813,11 @@ token {w} is repeated, and uppercase operand "w" holds "w" to one draw per expan
 
 The target is about a microsecond per value (goal 12).
 
-Each file is parsed, validated and weight-indexed once, in `New`, bar a table's name and
+Each file is parsed, validated and weight-indexed once, bar a table's name and parent
-parent columns, which are mapped on the first `Fake` that selects by name or descends
+columns, which are mapped on the first `Fake` that selects by name or descends through
-through it. Proving the draw fences adds a pass over the loaded tree per
+it. With the shipped set alone, a category loads on the first call reaching it, with
+every category it reads, and `List` reads an index; beside a `--data-path`, every
+category loads in `New`. Proving the draw fences adds a pass over the loaded tree per
 fence, and walks what a render reads only where data holds a reference, so a set that
 holds none pays for the passes alone. A `Fake` call then costs about what its output
 costs: an unweighted pick is O(1) whatever the list's length, a weighted one
@@ -931,7 +933,9 @@ Where the project is heading; the sections before Audience document what ships t
 
 ```
 doc.go          the package doc, and the vocabulary the package is written in
-fejkdata.go     Generator, New, options, the embedded data set, List
+fejkdata.go     Generator, New, options, List
+shipped.go      the embedded data set, and loading a shipped category on the first call reaching it
+shippedindex.go the index of shipped categories, generated from data/
 node.go         the node model and JSON -> node compilation, with its fence on a nested drawGroup
 table.go        tables: the rows TSV, its options and links, row selection and draws
 name.go         names: {ref as n} bindings, the scopes they live in, and the reads they answer
@@ -1016,7 +1020,7 @@ what the score asks of a pull request. Goal 5 is checked against simulated templ
 writers from the Audience.
 
 A change to the shipped data re-pins [`testdata/shipped_shape.txt`](testdata/shipped_shape.txt)
-in its own commit:
+and regenerates [`shippedindex.go`](shippedindex.go) in its own commit:
 
 ```sh
 REPIN=1 docker compose run --rm --user "$(id -u):$(id -g)" test

data.go

@@ -55,7 +55,7 @@ func loadData(sources []dataSource) (map[string]node, error) {
 	if len(root) == 0 {
 		return nil, fmt.Errorf("no .json data found")
 	}
-	if err := treeBinding(root).bind(); err != nil {
+	if err := categoryBinding(categorySites(&folder{children: root}), root).bind(); err != nil {
 		return nil, err
 	}
 	return root, nil
@@ -69,12 +69,7 @@ func loadDir(src dataSource, dir string) (*folder, error) {
 		return nil, fmt.Errorf("%s: %w", src.labelled(dir), err)
 	}
 	g := &folder{children: map[string]node{}}
-	files := &categoryFiles{src: src, dir: dir, tsv: map[string]bool{}}
+	files := newCategoryFiles(src, dir, entries)
-	for _, e := range entries {
-		if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() {
-			files.tsv[e.Name()] = false
-		}
-	}
 	for _, e := range entries {
 		if strings.HasPrefix(e.Name(), ".") { // hidden: a checkout or an editor's file, never data
 			continue
@@ -106,6 +101,16 @@ type categoryFiles struct {
 	tsv map[string]bool
 }
 
+func newCategoryFiles(src dataSource, dir string, entries []fs.DirEntry) *categoryFiles {
+	files := &categoryFiles{src: src, dir: dir, tsv: map[string]bool{}}
+	for _, e := range entries {
+		if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() {
+			files.tsv[e.Name()] = false
+		}
+	}
+	return files
+}
+
 func (c *categoryFiles) readRows(name string) (string, error) {
 	if _, present := c.tsv[name]; !present {
 		return "", fmt.Errorf("rows names %s, which is not beside it in %s", name, c.src.labelled(c.dir))
@@ -175,12 +180,45 @@ func mergeChildren(dst, src map[string]node) {
 	}
 }
 
-// nodeScope is the set of nodes one validation pass covers: a whole loaded tree,
+// nodeScope is the set of nodes one validation pass covers: the categories one load
-// or a single inline node.
+// binds, or a single inline node.
 type nodeScope func(fn func(path string, n node) error) error
 
-func treeScope(root map[string]node) nodeScope {
+// categorySite is a loaded category and where it sits: the folder holding it, that
-	return func(fn func(path string, n node) error) error { return walkNodes(root, fn) }
+// folder's path, and its own dot path.
+type categorySite struct {
+	dir  []string
+	path string
+	in   *folder
+	n    node
+}
+
+// categorySites lists every category under root, folders and names in sorted order.
+func categorySites(root *folder) []categorySite {
+	var out []categorySite
+	var walk func(dir []string, g *folder)
+	walk = func(dir []string, g *folder) {
+		for _, name := range sortedNames(g.children) {
+			if sub, isFolder := g.children[name].(*folder); isFolder {
+				walk(append(dir[:len(dir):len(dir)], name), sub)
+				continue
+			}
+			out = append(out, categorySite{dir: dir, path: join(strings.Join(dir, "."), name), in: g, n: g.children[name]})
+		}
+	}
+	walk(nil, root)
+	return out
+}
+
+func sitesScope(sites []categorySite) nodeScope {
+	return func(fn func(path string, n node) error) error {
+		for _, s := range sites {
+			if err := eachNode(s.n, s.path, fn); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
 }
 
 func inlineScope(n node, label string) nodeScope {
@@ -197,17 +235,19 @@ type binding struct {
 	typedByGo bool
 }
 
-func treeBinding(root map[string]node) binding {
+// categoryBinding binds the categories of one load, their references resolving
+// against root.
+func categoryBinding(sites []categorySite, root map[string]node) binding {
 	return binding{
-		scope: treeScope(root),
+		scope: sitesScope(sites),
 		link: func() error {
-			setTablePaths(root)
+			setTablePaths(sites)
-			if err := linkTables(root); err != nil {
+			if err := linkTables(sites); err != nil {
 				return err
 			}
-			return linkRefs(root)
+			return linkRefs(sites, root)
 		},
-		scopeFence: func() error { return checkNoCycles(root) },
+		scopeFence: func() error { return checkNoCycles(sites) },
 	}
 }
 
@@ -229,7 +269,7 @@ func (b binding) bind() error {
 // checkNodeFences runs the per-node fences every binding needs over a scope, each
 // over the whole scope before the next, so which of several broken nodes is reported
 // does not depend on the walk. Its walks recurse unguarded, so the scope's cycles are
-// refused first: a tree by checkNoCycles, and an inline node closes none, since nothing
+// refused first: categories by checkNoCycles, and an inline node closes none, since nothing
 // references it.
 func checkNodeFences(s nodeScope) error {
 	mem := renderCounts{}

docs/decisions.md

@@ -114,8 +114,19 @@ and `regexp.Match`, not two spellings of one result.
 A directory a machine happens to have would make `--seed 42` machine-dependent. Data
 still lives in `data/` as JSON; `--data-path` layers over it.
 
-`todo.md` items 76 and 77 revise it: shipped data loads on demand from an index built at
+`todo.md` item 77 revises it: a library imports the shipped data as Go modules.
-build time, and a library imports it as Go modules.
+
+## The shipped set alone loads a category on the first call reaching it; beside a `--data-path` it loads whole
+
+2026-10-03, approved by the maintainer as the restructure of `todo.md` item 76. Goal 12: a
+`New` pays nothing for the categories a run never reaches, which matters once the shipped
+set grows to hundreds of MB. A category loads with every category it references and its
+whole table family, and is bound as a whole load binds, so it renders as it would from
+`New`. `List` reads `shippedindex.go`, generated from `data/` with `REPIN=1`, and
+`TestShippedIndexIsCurrent` fails while it is stale. CI loads the whole shipped set, so a
+shipped category never fails at first reach. Goal 6: beside a `--data-path` or a
+`WithDataFS`, every category loads in `New`, so each mistake in a source the user wrote is
+`New`'s error. Valid while the shipped set is proven whole in CI.
 
 ## A bare reference draws each time; a reference path is held
 

drawfence_test.go

@@ -31,7 +31,7 @@ type fenceRoot struct {
 func fenceRoots(t *testing.T, f *Generator) []fenceRoot {
 	t.Helper()
 	var roots []fenceRoot
-	_ = walkNodes(f.root.children, func(path string, n node) error {
+	_ = sitesScope(categorySites(&f.root))(func(path string, n node) error {
 		if tm, ok := n.(*template); ok {
 			roots = append(roots, fenceRoot{path, tm})
 		}
@@ -58,7 +58,7 @@ func TestPairsFindWhatAWholeReplayFinds(t *testing.T) {
 	corpus := fenceCorpus(t)
 	roots := fenceRoots(t, corpus)
 	shipped := newShippedWhole(t)
-	_ = walkNodes(shipped.root.children, func(path string, n node) error {
+	_ = sitesScope(categorySites(&shipped.root))(func(path string, n node) error {
 		if tm, ok := n.(*template); ok {
 			roots = append(roots, fenceRoot{path, tm})
 		}
@@ -84,7 +84,7 @@ func TestPairsFindWhatAWholeReplayOfSelectedRowsFinds(t *testing.T) {
 	}
 	var reads []pathRead
 	fold := newReadFold()
-	_ = walkNodes(f.root.children, func(_ string, n node) error {
+	_ = sitesScope(categorySites(&f.root))(func(_ string, n node) error {
 		tb, isTable := n.(*table)
 		if !isTable || tb.keyIndex < 0 || tb.familyRoot().path != "region" {
 			return nil
@@ -190,7 +190,7 @@ func TestReadSurveyReportsAPinClashAsAnError(t *testing.T) {
 	if err != nil {
 		t.Fatalf("loadDir = %v", err)
 	}
-	if err := treeBinding(g.children).link(); err != nil {
+	if err := categoryBinding(categorySites(g), g.children).link(); err != nil {
 		t.Fatalf("link = %v", err)
 	}
 	r, isTemplate := g.children["r"].(*template)

fejkdata.go

@@ -2,7 +2,6 @@ package fejkdata
 
 import (
 	crand "crypto/rand"
-	"embed"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -14,9 +13,6 @@ import (
 	"sync"
 )
 
-//go:embed data
-var shippedFS embed.FS
-
 // MaxRepeat caps a repeat, and caps the renders nested repeats multiply to along
 // any one path; the CLI's --repeat shares it.
 const MaxRepeat = 1 << 20
@@ -30,9 +26,8 @@ var ErrNoData = errors.New("no data: WithoutShippedData needs at least one WithD
 // It is safe for concurrent use; a seeded sequence is reproducible only when drawn
 // from one goroutine.
 type Generator struct {
-	// mu guards rand, records and structs. The tree under root is fixed after New bar each
+	// mu guards rand, records, structs and root, which loads a shipped category on the
-	// table's row lookup, built under its sync.Once, so List, NewTemplate and
+	// first call reaching it.
-	// NewRecordTemplate read it without mu.
 	mu      sync.Mutex
 	rand    *generatorState
 	root    folder // the categories as the node a path walks from, owned here so a walk allocates none
@@ -97,15 +92,7 @@ func New(opts ...Option) (*Generator, error) {
 	for _, opt := range opts {
 		opt(&c)
 	}
-	var sources []dataSource
+	root, err := c.load()
-	if c.shipped {
-		sources = append(sources, dataSource{fsys: shippedFS, baseDir: "data"})
-	}
-	sources = append(sources, c.sources...)
-	if len(sources) == 0 {
-		return nil, fmt.Errorf("fejkdata: %w", ErrNoData)
-	}
-	cats, err := loadData(sources)
 	if err != nil {
 		return nil, fmt.Errorf("fejkdata: %w", err)
 	}
@@ -113,19 +100,35 @@ func New(opts ...Option) (*Generator, error) {
 	if err != nil {
 		return nil, fmt.Errorf("fejkdata: %w", err)
 	}
-	return &Generator{rand: rng, root: folder{children: cats}}, nil
+	return &Generator{rand: rng, root: root}, nil
+}
+
+// load is the tree New starts from. The shipped data set alone loads each category on the
+// first call reaching it; beside any other source, every category loads here, so each
+// mistake in that source is New's error.
+func (c config) load() (folder, error) {
+	if c.shipped && len(c.sources) == 0 {
+		return shelve(), nil
+	}
+	var sources []dataSource
+	if c.shipped {
+		sources = append(sources, shippedSource)
+	}
+	sources = append(sources, c.sources...)
+	if len(sources) == 0 {
+		return folder{}, ErrNoData
+	}
+	cats, err := loadData(sources)
+	return folder{children: cats}, err
 }
 
 // List returns the sorted dotted paths Fake renders: each category and every field,
 // column and linked table below it, a direct descent at a time. A choice consumes no
 // segment, so a path continues through one only where every variant carries it.
 func (f *Generator) List() []string {
-	var out []string
+	f.mu.Lock()
-	for _, name := range sortedNames(f.root.children) {
+	defer f.mu.Unlock()
-		for _, p := range paths(f.root.children[name]) {
+	out := paths(&f.root)
-			out = append(out, join(name, p))
-		}
-	}
 	sort.Strings(out)
 	return out
 }
@@ -141,6 +144,11 @@ func paths(n node) []string {
 				out = append(out, join(name, p))
 			}
 		}
+		for name, c := range n.unloaded {
+			for _, p := range c.paths {
+				out = append(out, join(name, p))
+			}
+		}
 		return out
 	case *template:
 		out := []string{""}

graph.go

@@ -5,16 +5,6 @@ import (
 	"sort"
 )
 
-// walkNodes calls fn once per contained node, passing the dot path that reaches it.
-func walkNodes(root map[string]node, fn func(path string, n node) error) error {
-	for _, name := range sortedNames(root) {
-		if err := eachNode(root[name], name, fn); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 // eachNode visits n and every node contained within it once, passing the dot path
 // that reaches each. It never crosses a reference edge — a {/path} reference is
 // skipped — so a single inline node is walked on its own.
@@ -196,7 +186,7 @@ func repeatCheck(path string, n node, mem renderCounts) error {
 // graph (renderEdges). Every node is a root: a field its parent's format never
 // renders is still reachable by dot path, so a cycle in one would otherwise reach
 // render and be fatal there.
-func checkNoCycles(root map[string]node) error {
+func checkNoCycles(sites []categorySite) error {
 	const (
 		grey  = 1
 		black = 2
@@ -219,5 +209,5 @@ func checkNoCycles(root map[string]node) error {
 		color[n] = black
 		return nil
 	}
-	return walkNodes(root, func(path string, n node) error { return visit(n, path) })
+	return sitesScope(sites)(func(path string, n node) error { return visit(n, path) })
 }

inline.go

@@ -32,7 +32,9 @@ func (f *Generator) NewTemplate(input string) (*Template, error) {
 	if err != nil {
 		return nil, fmt.Errorf("fejkdata: %w", err)
 	}
-	if err := inlineBinding(n, "template", f.root.children).bind(); err != nil {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := inlineBinding(n, "template", &f.root).bind(); err != nil {
 		return nil, fmt.Errorf("fejkdata: %w", err)
 	}
 	return &Template{g: f, n: n}, nil
@@ -146,11 +148,16 @@ func inputValue(input string) (any, error) {
 	return raw, nil
 }
 
-func inlineBinding(n node, label string, root map[string]node) binding {
+func inlineBinding(n node, label string, root *folder) binding {
 	scope := inlineScope(n, label)
 	return binding{
 		scope: scope,
-		link:  func() error { return linkNodeRefs(scope, root) },
+		link: func() error {
+			if err := loadShipped(root, shippedReads(root, nil, scope)); err != nil {
+				return err
+			}
+			return linkNodeRefs(scope, root.children)
+		},
 		scopeFence: func() error {
 			if t, isTemplate := n.(*template); isTemplate && t.drawGroup != "" {
 				return fmt.Errorf("%s: drawGroup %q names nothing, since nothing can reference an inline template; drop it", label, t.drawGroup)

node.go

@@ -16,7 +16,10 @@ type node interface{ isNode() }
 // folder is a namespace of named children, built from a directory of JSON files
 // and subdirectories. It has no value of its own: descend into a named child by
 // dot path; rendering one is an error (see Fake).
-type folder struct{ children map[string]node }
+type folder struct {
+	children map[string]node
+	unloaded map[string]shippedCategory // the shipped categories here a call has not reached yet
+}
 
 func (*folder) isNode() {}
 

reads_test.go

@@ -118,7 +118,7 @@ func TestReadFoldWalksACellByEveryRoute(t *testing.T) {
 	if err != nil {
 		t.Fatalf("loadDir = %v", err)
 	}
-	if err := treeBinding(g.children).link(); err != nil {
+	if err := categoryBinding(categorySites(g), g.children).link(); err != nil {
 		t.Fatalf("link = %v", err)
 	}
 	r, isTemplate := g.children["r"].(*template)

record.go

@@ -126,6 +126,9 @@ func (f *Generator) FakeRecord(path string) (*Record, error) {
 	if err != nil {
 		return nil, fmt.Errorf("fejkdata: %w", err)
 	}
+	if err := f.loadShippedAt(segments); err != nil {
+		return nil, fmt.Errorf("fejkdata: %w", err)
+	}
 	_, n, tail, err := resolveCategory(f.root.children, segments)
 	if err != nil {
 		return nil, fmt.Errorf("fejkdata: %s: %w", path, err)

reference.go

@@ -71,17 +71,16 @@ func refSegments(name string, folder []string) ([]string, error) {
 // so a reference sees the override-resolved tree. A reference's head binds into
 // refHeads under its root path and its tail reads like a sibling path, so two
 // spellings of one target are one draw.
-func linkRefs(root map[string]node) error {
+func linkRefs(sites []categorySite, root map[string]node) error {
-	if err := eachTemplate(root, func(folder []string, path string, t *template) error {
+	if err := eachTemplate(sites, func(s categorySite, path string, t *template) error {
-		category := strings.Join(strings.Split(path, ".")[:len(folder)+1], ".")
+		return linkTemplate(s.dir, path, s.path, t, root)
-		return linkTemplate(folder, path, category, t, root)
 	}); err != nil {
 		return err
 	}
-	if err := eachTemplate(root, func(_ []string, path string, t *template) error { return linkNames(path, t) }); err != nil {
+	if err := eachTemplate(sites, func(_ categorySite, path string, t *template) error { return linkNames(path, t) }); err != nil {
 		return err
 	}
-	return eachTemplate(root, func(_ []string, path string, t *template) error { return checkNameReads(path, t) })
+	return eachTemplate(sites, func(_ categorySite, path string, t *template) error { return checkNameReads(path, t) })
 }
 
 // linkTemplate resolves t's references in category, "" for an inline template, and
@@ -196,40 +195,20 @@ func loneRef(toks []formatToken) (string, bool) {
 	return toks[0].names[0], true
 }
 
-// eachTemplate calls fn once per template, with the folder its category sits in
+// eachTemplate calls fn once per template of the categories, with the category it sits
-// and the dot path reaching it, folders and names in sorted order.
+// in and the dot path reaching it.
-func eachTemplate(root map[string]node, fn func(folder []string, path string, t *template) error) error {
+func eachTemplate(sites []categorySite, fn func(s categorySite, path string, t *template) error) error {
-	var inCategory func(folder []string, path string, n node) error
+	for _, s := range sites {
-	inCategory = func(folder []string, path string, n node) error {
+		if err := eachNode(s.n, s.path, func(path string, n node) error {
-		if t, ok := n.(*template); ok {
+			if t, isTemplate := n.(*template); isTemplate {
-			if err := fn(folder, path, t); err != nil {
+				return fn(s, path, t)
-				return err
 			}
+			return nil
+		}); err != nil {
+			return err
 		}
-		for _, c := range contained(n) {
-			if err := inCategory(folder, join(path, c.name), c.node); err != nil {
-				return err
-			}
-		}
-		return nil
 	}
-	var inFolder func(dir []string, children map[string]node) error
+	return nil
-	inFolder = func(dir []string, children map[string]node) error {
-		for _, name := range sortedNames(children) {
-			path := join(strings.Join(dir, "."), name)
-			if g, ok := children[name].(*folder); ok {
-				if err := inFolder(append(dir[:len(dir):len(dir)], name), g.children); err != nil {
-					return err
-				}
-				continue
-			}
-			if err := inCategory(dir, path, children[name]); err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-	return inFolder(nil, root)
 }
 
 // resolveCategory walks a dotted path through the folders to the category it

render.go

@@ -24,6 +24,9 @@ func (f *Generator) Fake(path string) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("fejkdata: %w", err)
 	}
+	if err := f.loadShippedAt(segments); err != nil {
+		return "", fmt.Errorf("fejkdata: %w", err)
+	}
 	var draws renderDraws
 	sc := renderScope{draws: &draws}
 	n, err := descend(f.rand, &f.root, segments, sc)

shipped.go

@@ -0,0 +1,168 @@
+package fejkdata
+
+import (
+	"embed"
+	"fmt"
+	"io/fs"
+	"path"
+	"sort"
+	"strings"
+)
+
+//go:embed data
+var shippedFS embed.FS
+
+var shippedSource = dataSource{fsys: shippedFS, baseDir: "data"}
+
+// shippedCategory is a shipped category as shippedindex.go lists it: the table beside it
+// that it links to, "" for none, and the paths List advertises below it.
+type shippedCategory struct {
+	parent string
+	paths  []string
+}
+
+// shelve builds the folders of the shipped index, with every category in them unloaded.
+func shelve() folder {
+	root := folder{children: map[string]node{}}
+	for p, c := range shippedIndex {
+		segs := strings.Split(p, ".")
+		g := &root
+		for _, seg := range segs[:len(segs)-1] {
+			sub, isFolder := g.children[seg].(*folder)
+			if !isFolder {
+				sub = &folder{children: map[string]node{}}
+				g.children[seg] = sub
+			}
+			g = sub
+		}
+		if g.unloaded == nil {
+			g.unloaded = map[string]shippedCategory{}
+		}
+		g.unloaded[segs[len(segs)-1]] = c
+	}
+	return root
+}
+
+// pendingLoad is an unloaded shipped category: the folder holding it, that folder's path,
+// and its name there.
+type pendingLoad struct {
+	dir  []string
+	in   *folder
+	name string
+}
+
+// unloadedAt is the unloaded shipped category a path from root names, or descends into.
+func unloadedAt(root *folder, segs []string) (pendingLoad, bool) {
+	g := root
+	for i, seg := range segs {
+		switch n := g.children[seg].(type) {
+		case *folder:
+			g = n
+			continue
+		case nil:
+			if _, unloaded := g.unloaded[seg]; unloaded {
+				return pendingLoad{dir: segs[:i:i], in: g, name: seg}, true
+			}
+		}
+		return pendingLoad{}, false
+	}
+	return pendingLoad{}, false
+}
+
+// loadShippedAt loads the shipped category a caller's path names or descends into, unless
+// it is loaded already.
+func (f *Generator) loadShippedAt(segs []string) error {
+	if p, unloaded := unloadedAt(&f.root, segs); unloaded {
+		return loadShipped(&f.root, []pendingLoad{p})
+	}
+	return nil
+}
+
+// shippedReads is every unloaded shipped category the templates of scope reference, each
+// reference read from the folder dir.
+func shippedReads(root *folder, dir []string, scope nodeScope) []pendingLoad {
+	var out []pendingLoad
+	_ = scope(func(_ string, n node) error {
+		t, isTemplate := n.(*template)
+		if !isTemplate {
+			return nil
+		}
+		for _, name := range refTokens(t.tokens) {
+			segs, err := refSegments(name, dir)
+			if err != nil {
+				continue // the link reports it
+			}
+			if p, unloaded := unloadedAt(root, segs); unloaded {
+				out = append(out, p)
+			}
+		}
+		return nil
+	})
+	return out
+}
+
+// loadShipped loads the shipped categories wanted, with every category they read and the
+// whole table family of each, then binds what it loaded, as New binds a whole load. A failed
+// bind leaves every category it loaded unloaded again.
+func loadShipped(root *folder, wanted []pendingLoad) (err error) {
+	var loaded []pendingLoad
+	var entries []shippedCategory
+	defer func() {
+		if err == nil {
+			return
+		}
+		for i, p := range loaded {
+			delete(p.in.children, p.name)
+			p.in.unloaded[p.name] = entries[i]
+		}
+	}()
+	var sites []categorySite
+	for queue := wanted; len(queue) > 0; queue = queue[1:] {
+		p := queue[0]
+		c, unloaded := p.in.unloaded[p.name]
+		if !unloaded {
+			continue
+		}
+		site, err := p.load()
+		if err != nil {
+			return err
+		}
+		loaded, entries = append(loaded, p), append(entries, c)
+		sites = append(sites, site)
+		queue = append(queue, shippedReads(root, p.dir, sitesScope([]categorySite{site}))...)
+		queue = append(queue, p.family(c)...)
+	}
+	sort.Slice(sites, func(i, j int) bool { return sites[i].path < sites[j].path })
+	return categoryBinding(sites, root.children).bind()
+}
+
+// load parses and compiles the category, and moves it from its folder's unloaded
+// categories to its children.
+func (p pendingLoad) load() (categorySite, error) {
+	dir := path.Join(append([]string{shippedSource.baseDir}, p.dir...)...)
+	entries, err := fs.ReadDir(shippedSource.fsys, dir)
+	if err != nil {
+		return categorySite{}, fmt.Errorf("%s: %w", dir, err)
+	}
+	file := p.name + ".json"
+	if err := loadFile(shippedSource, p.in, path.Join(dir, file), file, newCategoryFiles(shippedSource, dir, entries)); err != nil {
+		return categorySite{}, err
+	}
+	delete(p.in.unloaded, p.name)
+	return categorySite{dir: p.dir, path: join(strings.Join(p.dir, "."), p.name), in: p.in, n: p.in.children[p.name]}, nil
+}
+
+// family is the tables beside p that a table family links it to: its parent, and every
+// table whose parent it is.
+func (p pendingLoad) family(c shippedCategory) []pendingLoad {
+	var out []pendingLoad
+	if c.parent != "" {
+		out = append(out, pendingLoad{dir: p.dir, in: p.in, name: c.parent})
+	}
+	for name, sibling := range p.in.unloaded {
+		if sibling.parent == p.name {
+			out = append(out, pendingLoad{dir: p.dir, in: p.in, name: name})
+		}
+	}
+	return out
+}

shippedindex.go

@@ -0,0 +1,81 @@
+// Code generated by TestShippedIndexIsCurrent with REPIN=1. DO NOT EDIT.
+
+package fejkdata
+
+var shippedIndex = map[string]shippedCategory{
+	"en_US.address":           {paths: []string{"", "locality", "postal-code", "region", "street", "street-number"}},
+	"en_US.color":             {paths: []string{""}},
+	"en_US.company":           {paths: []string{"", "base", "suffix"}},
+	"en_US.date":              {paths: []string{""}},
+	"en_US.email":             {paths: []string{"", "domain", "local", "local.n"}},
+	"en_US.first-name":        {parent: "sex", paths: []string{"", "count", "name", "sex"}},
+	"en_US.ip":                {paths: []string{""}},
+	"en_US.itin":              {paths: []string{"", "area", "group", "serial"}},
+	"en_US.last-name":         {paths: []string{"", "count", "name"}},
+	"en_US.person":            {paths: []string{"", "first", "last", "prefix", "sex"}},
+	"en_US.phone":             {paths: []string{"", "area", "exch", "line"}},
+	"en_US.price":             {paths: []string{"", "amt", "cents"}},
+	"en_US.sentence":          {paths: []string{"", "adj", "noun", "prep", "verb"}},
+	"en_US.sex":               {paths: []string{"", "code", "first-name", "first-name.count", "first-name.name", "first-name.sex", "name", "title", "title.name", "title.sex", "title.share"}},
+	"en_US.ssn":               {paths: []string{"", "area", "group", "serial"}},
+	"en_US.time":              {paths: []string{""}},
+	"en_US.title":             {parent: "sex", paths: []string{"", "name", "sex", "share"}},
+	"en_US.url":               {paths: []string{"", "host", "path"}},
+	"en_US.username":          {paths: []string{"", "n"}},
+	"en_US.version":           {paths: []string{"", "n", "pre", "suffix"}},
+	"en_US.word":              {paths: []string{""}},
+	"geo.SE.address":          {paths: []string{"", "locality", "postal-code", "street", "street-number"}},
+	"geo.SE.locality":         {parent: "municipality", paths: []string{"", "lat", "lon", "municipality", "name", "population", "postal-code", "postal-code.code", "postal-code.locality", "street", "street.locality", "street.name", "street.segments"}},
+	"geo.SE.municipality":     {parent: "region", paths: []string{"", "code", "locality", "locality.lat", "locality.lon", "locality.municipality", "locality.name", "locality.population", "locality.postal-code", "locality.postal-code.code", "locality.postal-code.locality", "locality.street", "locality.street.locality", "locality.street.name", "locality.street.segments", "name", "population", "region"}},
+	"geo.SE.postal-code":      {parent: "locality", paths: []string{"", "code", "locality"}},
+	"geo.SE.region":           {paths: []string{"", "code", "municipality", "municipality.code", "municipality.locality", "municipality.locality.lat", "municipality.locality.lon", "municipality.locality.municipality", "municipality.locality.name", "municipality.locality.population", "municipality.locality.postal-code", "municipality.locality.postal-code.code", "municipality.locality.postal-code.locality", "municipality.locality.street", "municipality.locality.street.locality", "municipality.locality.street.name", "municipality.locality.street.segments", "municipality.name", "municipality.population", "municipality.region", "name", "population", "timezone"}},
+	"geo.SE.street":           {parent: "locality", paths: []string{"", "locality", "name", "segments"}},
+	"geo.US.address":          {paths: []string{"", "locality", "postal-code", "region", "street", "street-number"}},
+	"geo.US.locality":         {parent: "municipality", paths: []string{"", "code", "lat", "lon", "municipality", "name", "population", "postal-code", "postal-code.addresses", "postal-code.code", "postal-code.locality", "street", "street.addresses", "street.locality", "street.name"}},
+	"geo.US.municipality":     {parent: "region", paths: []string{"", "code", "locality", "locality.code", "locality.lat", "locality.lon", "locality.municipality", "locality.name", "locality.population", "locality.postal-code", "locality.postal-code.addresses", "locality.postal-code.code", "locality.postal-code.locality", "locality.street", "locality.street.addresses", "locality.street.locality", "locality.street.name", "name", "population", "region"}},
+	"geo.US.postal-code":      {parent: "locality", paths: []string{"", "addresses", "code", "locality"}},
+	"geo.US.region":           {paths: []string{"", "abbr", "code", "municipality", "municipality.code", "municipality.locality", "municipality.locality.code", "municipality.locality.lat", "municipality.locality.lon", "municipality.locality.municipality", "municipality.locality.name", "municipality.locality.population", "municipality.locality.postal-code", "municipality.locality.postal-code.addresses", "municipality.locality.postal-code.code", "municipality.locality.postal-code.locality", "municipality.locality.street", "municipality.locality.street.addresses", "municipality.locality.street.locality", "municipality.locality.street.name", "municipality.name", "municipality.population", "municipality.region", "name", "population", "timezone"}},
+	"geo.US.street":           {parent: "locality", paths: []string{"", "addresses", "locality", "name"}},
+	"misc.car":                {paths: []string{"", "make", "model"}},
+	"misc.coordinate":         {paths: []string{"", "lat", "lon"}},
+	"misc.creditcard":         {paths: []string{"", "d"}},
+	"misc.currency":           {paths: []string{"", "code", "decimals", "name", "numeric", "symbol"}},
+	"misc.datetime":           {paths: []string{""}},
+	"misc.emoji":              {paths: []string{""}},
+	"misc.httpmethod":         {paths: []string{"", "idempotent", "method", "safe"}},
+	"misc.httpstatus":         {paths: []string{"", "code", "reason"}},
+	"misc.language":           {paths: []string{"", "code", "code3", "name"}},
+	"misc.loglevel":           {paths: []string{"", "code", "keyword", "severity"}},
+	"misc.mac":                {paths: []string{""}},
+	"misc.mimetype":           {paths: []string{"", "ext", "type"}},
+	"misc.objectid":           {paths: []string{""}},
+	"misc.port":               {paths: []string{"", "number", "service"}},
+	"misc.protocol":           {paths: []string{"", "keyword", "name", "number"}},
+	"misc.territory":          {paths: []string{"", "alpha2", "alpha3", "calling-code", "capital", "country", "currency", "flag", "languages", "name", "numeric", "timezone", "timezone.offset", "timezone.territory", "timezone.weight", "timezone.zone", "tld"}},
+	"misc.timezone":           {parent: "territory", paths: []string{"", "offset", "territory", "weight", "zone"}},
+	"misc.tld":                {paths: []string{"", "tld", "type", "unicode"}},
+	"misc.useragent":          {paths: []string{"", "browser", "device", "os", "ua"}},
+	"misc.uuid":               {paths: []string{"", "variant"}},
+	"sv_SE.address":           {paths: []string{"", "locality", "postal-code", "street", "street-number"}},
+	"sv_SE.birth-number":      {parent: "sex", paths: []string{"", "number", "sex"}},
+	"sv_SE.color":             {paths: []string{""}},
+	"sv_SE.company":           {paths: []string{"", "base", "suffix"}},
+	"sv_SE.date":              {paths: []string{""}},
+	"sv_SE.email":             {paths: []string{"", "domain", "local", "local.n"}},
+	"sv_SE.first-name":        {parent: "sex", paths: []string{"", "count", "name", "sex"}},
+	"sv_SE.ip":                {paths: []string{""}},
+	"sv_SE.last-name":         {paths: []string{"", "count", "name"}},
+	"sv_SE.person":            {paths: []string{"", "first", "last", "prefix", "sex"}},
+	"sv_SE.personnummer":      {paths: []string{""}},
+	"sv_SE.phone":             {paths: []string{"", "a", "b", "c", "prefix"}},
+	"sv_SE.price":             {paths: []string{"", "amt", "ore"}},
+	"sv_SE.samordningsnummer": {paths: []string{""}},
+	"sv_SE.sentence":          {paths: []string{"", "adj", "noun", "prep", "verb"}},
+	"sv_SE.sex":               {paths: []string{"", "birth-number", "birth-number.number", "birth-number.sex", "code", "first-name", "first-name.count", "first-name.name", "first-name.sex", "name"}},
+	"sv_SE.time":              {paths: []string{""}},
+	"sv_SE.title":             {paths: []string{"", "name", "share"}},
+	"sv_SE.url":               {paths: []string{"", "host", "path"}},
+	"sv_SE.username":          {paths: []string{"", "n"}},
+	"sv_SE.version":           {paths: []string{"", "n", "pre", "suffix"}},
+	"sv_SE.word":              {paths: []string{""}},
+}

struct.go

@@ -49,7 +49,7 @@ func (f *Generator) structShapeOf(t reflect.Type) (*structShape, error) {
 	if label == "" {
 		label = "struct"
 	}
-	sc := &structCompile{categories: f.root.children, visiting: map[reflect.Type]bool{}, structsLeft: maxStructs}
+	sc := &structCompile{root: &f.root, visiting: map[reflect.Type]bool{}, structsLeft: maxStructs}
 	shape, err := sc.compileShape(t, label)
 	if err == nil && shape.empty() {
 		err = fmt.Errorf("%s has no fake tags, so nothing to fill", t)
@@ -82,7 +82,7 @@ func (s *structShape) empty() bool { return s.record == nil && len(s.nested) ==
 // loaded tree, the types compiling or embedded above, so a pointer back to one is left alone
 // rather than filled without end, and how many more structs it may walk.
 type structCompile struct {
-	categories  map[string]node
+	root        *folder
 	visiting    map[reflect.Type]bool
 	structsLeft int
 }
@@ -109,7 +109,7 @@ func (sc *structCompile) compileShape(t reflect.Type, label string) (*structShap
 		return nil, err
 	}
 	if len(c.tags) > 0 {
-		if err := c.shape.compileRecord(sc.categories, t, label, c.tags); err != nil {
+		if err := c.shape.compileRecord(sc.root, t, label, c.tags); err != nil {
 			return nil, err
 		}
 	}
@@ -252,13 +252,13 @@ func checkTaggedType(sf reflect.StructField) error {
 
 // compileRecord compiles the tagged fields of t as one record, and proves each column holds
 // only what its field's Go type can.
-func (s *structShape) compileRecord(categories map[string]node, t reflect.Type, label string, tags map[string]any) error {
+func (s *structShape) compileRecord(root *folder, t reflect.Type, label string, tags map[string]any) error {
 	tags["format"] = ""
 	n, err := compile(tags)
 	if err != nil {
 		return fmt.Errorf("%s: %w", label, err)
 	}
-	b := inlineBinding(n, label, categories)
+	b := inlineBinding(n, label, root)
 	b.typedByGo = true
 	if err := b.bind(); err != nil {
 		return err

table.go

@@ -365,47 +365,29 @@ func refuseTableBinding(toks []formatToken) error {
 
 // setTablePaths gives every table the path a selector on it is written at, before a
 // link or a draw can name one.
-func setTablePaths(root map[string]node) {
+func setTablePaths(sites []categorySite) {
-	var walk func(dir string, children map[string]node)
+	for _, s := range sites {
-	walk = func(dir string, children map[string]node) {
+		if t, isTable := s.n.(*table); isTable {
-		for name, child := range children {
+			t.path = s.path
-			switch n := child.(type) {
-			case *folder:
-				walk(join(dir, name), n.children)
-			case *table:
-				n.path = join(dir, name)
-			}
 		}
 	}
-	walk("", root)
 }
 
 // linkTables binds every table's parent to the table beside it, and proves the
 // links: a parent has a key, every link cell is one, every parent row is linked
 // to, no chain of parents closes, and no child is named like a parent's column.
 // docs/decisions.md#a-parent-row-with-no-child-row-is-a-load-error
-func linkTables(root map[string]node) error {
+func linkTables(sites []categorySite) error {
-	var walk func(dir string, children map[string]node) error
+	for _, s := range sites {
-	walk = func(dir string, children map[string]node) error {
+		t, isTable := s.n.(*table)
-		for _, name := range sortedNames(children) {
+		if !isTable || t.parentIndex < 0 {
-			path := join(dir, name)
+			continue
-			switch n := children[name].(type) {
+		}
-			case *folder:
+		if err := t.linkParent(s.path, s.in.children); err != nil {
-				if err := walk(path, n.children); err != nil {
+			return fmt.Errorf("%s: %w", s.path, err)
-					return err
-				}
-			case *table:
-				if n.parentIndex < 0 {
-					continue
-				}
-				if err := n.linkParent(path, children); err != nil {
-					return fmt.Errorf("%s: %w", path, err)
-				}
-			}
 		}
-		return nil
 	}
-	return walk("", root)
+	return nil
 }
 
 func (t *table) linkParent(path string, siblings map[string]node) error {