Load shipped categories before binding, panic on an index that misses a read, and name the index entry and unloaded category plainly

AGENTS.md

@@ -43,7 +43,7 @@ In [docs/decisions.md](docs/decisions.md):
 - A padded JSON argument is rejected, not trimmed
 - `FakeTemplate` and `NewTemplate` both stay
 - The shipped data is embedded, not discovered
-- The shipped set alone loads a category on the first call reaching it; beside a `--data-path` it loads whole
+- The shipped set alone loads a category on the first call reaching it; beside a `--data-path` every category loads in `New`
 - A bare reference draws each time; a reference path is held
 - Reference sigils follow the filesystem
 - A change to what exists is a major; a minor only adds

docs/decisions.md

@@ -116,17 +116,17 @@ still lives in `data/` as JSON; `--data-path` layers over it.
 
 `todo.md` item 77 revises it: a library imports the shipped data as Go modules.
 
-## The shipped set alone loads a category on the first call reaching it; beside a `--data-path` it loads whole
+## The shipped set alone loads a category on the first call reaching it; beside a `--data-path` every category loads in `New`
 
 2026-10-03, approved by the maintainer as the restructure of `todo.md` item 76. Goal 12: a
 `New` pays nothing for the categories a run never reaches, which matters once the shipped
 set grows to hundreds of MB. A category loads with every category it references and its
 whole table family, and is bound as a whole load binds, so it renders as it would from
 `New`. `List` reads `shippedindex.go`, generated from `data/` with `REPIN=1`, and
-`TestShippedIndexIsCurrent` fails while it is stale. CI loads the whole shipped set, so a
+`TestShippedIndexIsCurrent` fails while it is stale. CI loads the whole shipped set, and
-shipped category never fails at first reach. Goal 6: beside a `--data-path` or a
+loads each shipped category alone into a fresh `New`, so none fails at first reach. Goal 6: beside a `--data-path` or a
 `WithDataFS`, every category loads in `New`, so each mistake in a source the user wrote is
-`New`'s error. Valid while the shipped set is proven whole in CI.
+`New`'s error. Valid while CI proves the shipped set whole and each category alone.
 
 ## A bare reference draws each time; a reference path is held
 

fejkdata.go

@@ -108,7 +108,7 @@ func New(opts ...Option) (*Generator, error) {
 // mistake in that source is New's error.
 func (c config) load() (folder, error) {
 	if c.shipped && len(c.sources) == 0 {
-		return shelve(), nil
+		return unloadedTree(), nil
 	}
 	var sources []dataSource
 	if c.shipped {
@@ -144,8 +144,8 @@ func paths(n node) []string {
 				out = append(out, join(name, p))
 			}
 		}
-		for name, c := range n.unloaded {
+		for name, e := range n.unloaded {
-			for _, p := range c.paths {
+			for _, p := range e.paths {
 				out = append(out, join(name, p))
 			}
 		}

inline.go

@@ -34,7 +34,8 @@ func (f *Generator) NewTemplate(input string) (*Template, error) {
 	}
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	if err := inlineBinding(n, "template", &f.root).bind(); err != nil {
+	loadShipped(&f.root, unloadedReads(&f.root, nil, inlineScope(n, "template")))
+	if err := inlineBinding(n, "template", f.root.children).bind(); err != nil {
 		return nil, fmt.Errorf("fejkdata: %w", err)
 	}
 	return &Template{g: f, n: n}, nil
@@ -148,16 +149,11 @@ func inputValue(input string) (any, error) {
 	return raw, nil
 }
 
-func inlineBinding(n node, label string, root *folder) binding {
+func inlineBinding(n node, label string, root map[string]node) binding {
 	scope := inlineScope(n, label)
 	return binding{
 		scope: scope,
-		link: func() error {
+		link:  func() error { return linkNodeRefs(scope, root) },
-			if err := loadShipped(root, shippedReads(root, nil, scope)); err != nil {
-				return err
-			}
-			return linkNodeRefs(scope, root.children)
-		},
 		scopeFence: func() error {
 			if t, isTemplate := n.(*template); isTemplate && t.drawGroup != "" {
 				return fmt.Errorf("%s: drawGroup %q names nothing, since nothing can reference an inline template; drop it", label, t.drawGroup)

node.go

@@ -18,7 +18,7 @@ type node interface{ isNode() }
 // dot path; rendering one is an error (see Fake).
 type folder struct {
 	children map[string]node
-	unloaded map[string]shippedCategory // the shipped categories here a call has not reached yet
+	unloaded map[string]shippedEntry // the shipped categories here a call has not reached yet
 }
 
 func (*folder) isNode() {}

record.go

@@ -126,9 +126,7 @@ func (f *Generator) FakeRecord(path string) (*Record, error) {
 	if err != nil {
 		return nil, fmt.Errorf("fejkdata: %w", err)
 	}
-	if err := f.loadShippedAt(segments); err != nil {
+	f.loadShippedAt(segments)
-		return nil, fmt.Errorf("fejkdata: %w", err)
-	}
 	_, n, tail, err := resolveCategory(f.root.children, segments)
 	if err != nil {
 		return nil, fmt.Errorf("fejkdata: %s: %w", path, err)

render.go

@@ -24,9 +24,7 @@ func (f *Generator) Fake(path string) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("fejkdata: %w", err)
 	}
-	if err := f.loadShippedAt(segments); err != nil {
+	f.loadShippedAt(segments)
-		return "", fmt.Errorf("fejkdata: %w", err)
-	}
 	var draws renderDraws
 	sc := renderScope{draws: &draws}
 	n, err := descend(f.rand, &f.root, segments, sc)

shipped.go

@@ -14,17 +14,17 @@ var shippedFS embed.FS
 
 var shippedSource = dataSource{fsys: shippedFS, baseDir: "data"}
 
-// shippedCategory is a shipped category as shippedindex.go lists it: the table beside it
+// shippedEntry is a shipped category's entry in shippedindex.go: the table beside it that
-// that it links to, "" for none, and the paths List advertises below it.
+// it links to, "" for none, and the paths List advertises below it.
-type shippedCategory struct {
+type shippedEntry struct {
 	parent string
 	paths  []string
 }
 
-// shelve builds the folders of the shipped index, with every category in them unloaded.
+// unloadedTree builds the folders of the shipped index, with every category in them unloaded.
-func shelve() folder {
+func unloadedTree() folder {
 	root := folder{children: map[string]node{}}
-	for p, c := range shippedIndex {
+	for p, e := range shippedIndex {
 		segs := strings.Split(p, ".")
 		g := &root
 		for _, seg := range segs[:len(segs)-1] {
@@ -36,23 +36,23 @@ func shelve() folder {
 			g = sub
 		}
 		if g.unloaded == nil {
-			g.unloaded = map[string]shippedCategory{}
+			g.unloaded = map[string]shippedEntry{}
 		}
-		g.unloaded[segs[len(segs)-1]] = c
+		g.unloaded[segs[len(segs)-1]] = e
 	}
 	return root
 }
 
-// pendingLoad is an unloaded shipped category: the folder holding it, that folder's path,
+// unloadedCategory is a shipped category not loaded yet: the folder holding it, that
-// and its name there.
+// folder's path, and its name there.
-type pendingLoad struct {
+type unloadedCategory struct {
 	dir  []string
 	in   *folder
 	name string
 }
 
 // unloadedAt is the unloaded shipped category a path from root names, or descends into.
-func unloadedAt(root *folder, segs []string) (pendingLoad, bool) {
+func unloadedAt(root *folder, segs []string) (unloadedCategory, bool) {
 	g := root
 	for i, seg := range segs {
 		switch n := g.children[seg].(type) {
@@ -61,27 +61,26 @@ func unloadedAt(root *folder, segs []string) (pendingLoad, bool) {
 			continue
 		case nil:
 			if _, unloaded := g.unloaded[seg]; unloaded {
-				return pendingLoad{dir: segs[:i:i], in: g, name: seg}, true
+				return unloadedCategory{dir: segs[:i:i], in: g, name: seg}, true
 			}
 		}
-		return pendingLoad{}, false
+		return unloadedCategory{}, false
 	}
-	return pendingLoad{}, false
+	return unloadedCategory{}, false
 }
 
 // loadShippedAt loads the shipped category a caller's path names or descends into, unless
 // it is loaded already.
-func (f *Generator) loadShippedAt(segs []string) error {
+func (f *Generator) loadShippedAt(segs []string) {
-	if p, unloaded := unloadedAt(&f.root, segs); unloaded {
+	if u, unloaded := unloadedAt(&f.root, segs); unloaded {
-		return loadShipped(&f.root, []pendingLoad{p})
+		loadShipped(&f.root, []unloadedCategory{u})
 	}
-	return nil
 }
 
-// shippedReads is every unloaded shipped category the templates of scope reference, each
+// unloadedReads is every unloaded shipped category the templates of scope reference, each
 // reference read from the folder dir.
-func shippedReads(root *folder, dir []string, scope nodeScope) []pendingLoad {
+func unloadedReads(root *folder, dir []string, scope nodeScope) []unloadedCategory {
-	var out []pendingLoad
+	var out []unloadedCategory
 	_ = scope(func(_ string, n node) error {
 		t, isTemplate := n.(*template)
 		if !isTemplate {
@@ -92,8 +91,8 @@ func shippedReads(root *folder, dir []string, scope nodeScope) []pendingLoad {
 			if err != nil {
 				continue // the link reports it
 			}
-			if p, unloaded := unloadedAt(root, segs); unloaded {
+			if u, unloaded := unloadedAt(root, segs); unloaded {
-				out = append(out, p)
+				out = append(out, u)
 			}
 		}
 		return nil
@@ -102,66 +101,59 @@ func shippedReads(root *folder, dir []string, scope nodeScope) []pendingLoad {
 }
 
 // loadShipped loads the shipped categories wanted, with every category they read and the
-// whole table family of each, then binds what it loaded, as New binds a whole load. A failed
+// whole table family of each, then binds what it loaded, as New binds a whole load.
-// bind leaves every category it loaded unloaded again.
+// TestEveryShippedCategoryLoadsAlone proves each closure, so a failure is the index's.
-func loadShipped(root *folder, wanted []pendingLoad) (err error) {
+func loadShipped(root *folder, wanted []unloadedCategory) {
-	var loaded []pendingLoad
-	var entries []shippedCategory
-	defer func() {
-		if err == nil {
-			return
-		}
-		for i, p := range loaded {
-			delete(p.in.children, p.name)
-			p.in.unloaded[p.name] = entries[i]
-		}
-	}()
 	var sites []categorySite
 	for queue := wanted; len(queue) > 0; queue = queue[1:] {
-		p := queue[0]
+		u := queue[0]
-		c, unloaded := p.in.unloaded[p.name]
+		e, unloaded := u.in.unloaded[u.name]
 		if !unloaded {
 			continue
 		}
-		site, err := p.load()
+		site, err := u.load()
 		if err != nil {
-			return err
+			panic(internalError("shipped %s: %v", u.name, err))
 		}
-		loaded, entries = append(loaded, p), append(entries, c)
 		sites = append(sites, site)
-		queue = append(queue, shippedReads(root, p.dir, sitesScope([]categorySite{site}))...)
+		queue = append(queue, unloadedReads(root, u.dir, sitesScope([]categorySite{site}))...)
-		queue = append(queue, p.family(c)...)
+		queue = append(queue, u.family(e)...)
+	}
+	if len(sites) == 0 {
+		return
 	}
 	sort.Slice(sites, func(i, j int) bool { return sites[i].path < sites[j].path })
-	return categoryBinding(sites, root.children).bind()
+	if err := categoryBinding(sites, root.children).bind(); err != nil {
+		panic(internalError("binding shipped categories: %v", err))
+	}
 }
 
 // load parses and compiles the category, and moves it from its folder's unloaded
 // categories to its children.
-func (p pendingLoad) load() (categorySite, error) {
+func (u unloadedCategory) load() (categorySite, error) {
-	dir := path.Join(append([]string{shippedSource.baseDir}, p.dir...)...)
+	dir := path.Join(append([]string{shippedSource.baseDir}, u.dir...)...)
 	entries, err := fs.ReadDir(shippedSource.fsys, dir)
 	if err != nil {
 		return categorySite{}, fmt.Errorf("%s: %w", dir, err)
 	}
-	file := p.name + ".json"
+	file := u.name + ".json"
-	if err := loadFile(shippedSource, p.in, path.Join(dir, file), file, newCategoryFiles(shippedSource, dir, entries)); err != nil {
+	if err := loadFile(shippedSource, u.in, path.Join(dir, file), file, newCategoryFiles(shippedSource, dir, entries)); err != nil {
 		return categorySite{}, err
 	}
-	delete(p.in.unloaded, p.name)
+	delete(u.in.unloaded, u.name)
-	return categorySite{dir: p.dir, path: join(strings.Join(p.dir, "."), p.name), in: p.in, n: p.in.children[p.name]}, nil
+	return categorySite{dir: u.dir, path: join(strings.Join(u.dir, "."), u.name), in: u.in, n: u.in.children[u.name]}, nil
 }
 
-// family is the tables beside p that a table family links it to: its parent, and every
+// family is the tables beside u that a table family links it to: its parent, and every
 // table whose parent it is.
-func (p pendingLoad) family(c shippedCategory) []pendingLoad {
+func (u unloadedCategory) family(e shippedEntry) []unloadedCategory {
-	var out []pendingLoad
+	var out []unloadedCategory
-	if c.parent != "" {
+	if e.parent != "" {
-		out = append(out, pendingLoad{dir: p.dir, in: p.in, name: c.parent})
+		out = append(out, unloadedCategory{dir: u.dir, in: u.in, name: e.parent})
 	}
-	for name, sibling := range p.in.unloaded {
+	for name, sibling := range u.in.unloaded {
-		if sibling.parent == p.name {
+		if sibling.parent == u.name {
-			out = append(out, pendingLoad{dir: p.dir, in: p.in, name: name})
+			out = append(out, unloadedCategory{dir: u.dir, in: u.in, name: name})
 		}
 	}
 	return out

shippedindex.go

@@ -2,7 +2,7 @@
 
 package fejkdata
 
-var shippedIndex = map[string]shippedCategory{
+var shippedIndex = map[string]shippedEntry{
 	"en_US.address":           {paths: []string{"", "locality", "postal-code", "region", "street", "street-number"}},
 	"en_US.color":             {paths: []string{""}},
 	"en_US.company":           {paths: []string{"", "base", "suffix"}},

struct.go

@@ -258,7 +258,8 @@ func (s *structShape) compileRecord(root *folder, t reflect.Type, label string,
 	if err != nil {
 		return fmt.Errorf("%s: %w", label, err)
 	}
-	b := inlineBinding(n, label, root)
+	loadShipped(root, unloadedReads(root, nil, inlineScope(n, label)))
+	b := inlineBinding(n, label, root.children)
 	b.typedByGo = true
 	if err := b.bind(); err != nil {
 		return err