diff --git a/AGENTS.md b/AGENTS.md index b38477a..bc4a4a9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,7 +43,7 @@ In [docs/decisions.md](docs/decisions.md): - A padded JSON argument is rejected, not trimmed - `FakeTemplate` and `NewTemplate` both stay - The shipped data is embedded, not discovered -- The shipped set alone loads a category on the first call reaching it; beside a `--data-path` it loads whole +- The shipped set alone loads a category on the first call reaching it; beside a `--data-path` every category loads in `New` - A bare reference draws each time; a reference path is held - Reference sigils follow the filesystem - A change to what exists is a major; a minor only adds diff --git a/docs/decisions.md b/docs/decisions.md index 6c6159a..07c17d6 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -116,17 +116,17 @@ still lives in `data/` as JSON; `--data-path` layers over it. `todo.md` item 77 revises it: a library imports the shipped data as Go modules. -## The shipped set alone loads a category on the first call reaching it; beside a `--data-path` it loads whole +## The shipped set alone loads a category on the first call reaching it; beside a `--data-path` every category loads in `New` 2026-10-03, approved by the maintainer as the restructure of `todo.md` item 76. Goal 12: a `New` pays nothing for the categories a run never reaches, which matters once the shipped set grows to hundreds of MB. A category loads with every category it references and its whole table family, and is bound as a whole load binds, so it renders as it would from `New`. `List` reads `shippedindex.go`, generated from `data/` with `REPIN=1`, and -`TestShippedIndexIsCurrent` fails while it is stale. CI loads the whole shipped set, so a -shipped category never fails at first reach. Goal 6: beside a `--data-path` or a +`TestShippedIndexIsCurrent` fails while it is stale. CI loads the whole shipped set, and +loads each shipped category alone into a fresh `New`, so none fails at first reach. Goal 6: beside a `--data-path` or a `WithDataFS`, every category loads in `New`, so each mistake in a source the user wrote is -`New`'s error. Valid while the shipped set is proven whole in CI. +`New`'s error. Valid while CI proves the shipped set whole and each category alone. ## A bare reference draws each time; a reference path is held diff --git a/fejkdata.go b/fejkdata.go index 4dcacb2..8108780 100644 --- a/fejkdata.go +++ b/fejkdata.go @@ -108,7 +108,7 @@ func New(opts ...Option) (*Generator, error) { // mistake in that source is New's error. func (c config) load() (folder, error) { if c.shipped && len(c.sources) == 0 { - return shelve(), nil + return unloadedTree(), nil } var sources []dataSource if c.shipped { @@ -144,8 +144,8 @@ func paths(n node) []string { out = append(out, join(name, p)) } } - for name, c := range n.unloaded { - for _, p := range c.paths { + for name, e := range n.unloaded { + for _, p := range e.paths { out = append(out, join(name, p)) } } diff --git a/inline.go b/inline.go index 9547e08..f0adbd7 100644 --- a/inline.go +++ b/inline.go @@ -34,7 +34,8 @@ func (f *Generator) NewTemplate(input string) (*Template, error) { } f.mu.Lock() defer f.mu.Unlock() - if err := inlineBinding(n, "template", &f.root).bind(); err != nil { + loadShipped(&f.root, unloadedReads(&f.root, nil, inlineScope(n, "template"))) + if err := inlineBinding(n, "template", f.root.children).bind(); err != nil { return nil, fmt.Errorf("fejkdata: %w", err) } return &Template{g: f, n: n}, nil @@ -148,16 +149,11 @@ func inputValue(input string) (any, error) { return raw, nil } -func inlineBinding(n node, label string, root *folder) binding { +func inlineBinding(n node, label string, root map[string]node) binding { scope := inlineScope(n, label) return binding{ scope: scope, - link: func() error { - if err := loadShipped(root, shippedReads(root, nil, scope)); err != nil { - return err - } - return linkNodeRefs(scope, root.children) - }, + link: func() error { return linkNodeRefs(scope, root) }, scopeFence: func() error { if t, isTemplate := n.(*template); isTemplate && t.drawGroup != "" { return fmt.Errorf("%s: drawGroup %q names nothing, since nothing can reference an inline template; drop it", label, t.drawGroup) diff --git a/node.go b/node.go index 6a9c3f2..822ab3a 100644 --- a/node.go +++ b/node.go @@ -18,7 +18,7 @@ type node interface{ isNode() } // dot path; rendering one is an error (see Fake). type folder struct { children map[string]node - unloaded map[string]shippedCategory // the shipped categories here a call has not reached yet + unloaded map[string]shippedEntry // the shipped categories here a call has not reached yet } func (*folder) isNode() {} diff --git a/record.go b/record.go index 40b8c31..b459301 100644 --- a/record.go +++ b/record.go @@ -126,9 +126,7 @@ func (f *Generator) FakeRecord(path string) (*Record, error) { if err != nil { return nil, fmt.Errorf("fejkdata: %w", err) } - if err := f.loadShippedAt(segments); err != nil { - return nil, fmt.Errorf("fejkdata: %w", err) - } + f.loadShippedAt(segments) _, n, tail, err := resolveCategory(f.root.children, segments) if err != nil { return nil, fmt.Errorf("fejkdata: %s: %w", path, err) diff --git a/render.go b/render.go index 2d306cf..d08dc04 100644 --- a/render.go +++ b/render.go @@ -24,9 +24,7 @@ func (f *Generator) Fake(path string) (string, error) { if err != nil { return "", fmt.Errorf("fejkdata: %w", err) } - if err := f.loadShippedAt(segments); err != nil { - return "", fmt.Errorf("fejkdata: %w", err) - } + f.loadShippedAt(segments) var draws renderDraws sc := renderScope{draws: &draws} n, err := descend(f.rand, &f.root, segments, sc) diff --git a/shipped.go b/shipped.go index 1cabd5f..a7c7525 100644 --- a/shipped.go +++ b/shipped.go @@ -14,17 +14,17 @@ var shippedFS embed.FS var shippedSource = dataSource{fsys: shippedFS, baseDir: "data"} -// shippedCategory is a shipped category as shippedindex.go lists it: the table beside it -// that it links to, "" for none, and the paths List advertises below it. -type shippedCategory struct { +// shippedEntry is a shipped category's entry in shippedindex.go: the table beside it that +// it links to, "" for none, and the paths List advertises below it. +type shippedEntry struct { parent string paths []string } -// shelve builds the folders of the shipped index, with every category in them unloaded. -func shelve() folder { +// unloadedTree builds the folders of the shipped index, with every category in them unloaded. +func unloadedTree() folder { root := folder{children: map[string]node{}} - for p, c := range shippedIndex { + for p, e := range shippedIndex { segs := strings.Split(p, ".") g := &root for _, seg := range segs[:len(segs)-1] { @@ -36,23 +36,23 @@ func shelve() folder { g = sub } if g.unloaded == nil { - g.unloaded = map[string]shippedCategory{} + g.unloaded = map[string]shippedEntry{} } - g.unloaded[segs[len(segs)-1]] = c + g.unloaded[segs[len(segs)-1]] = e } return root } -// pendingLoad is an unloaded shipped category: the folder holding it, that folder's path, -// and its name there. -type pendingLoad struct { +// unloadedCategory is a shipped category not loaded yet: the folder holding it, that +// folder's path, and its name there. +type unloadedCategory struct { dir []string in *folder name string } // unloadedAt is the unloaded shipped category a path from root names, or descends into. -func unloadedAt(root *folder, segs []string) (pendingLoad, bool) { +func unloadedAt(root *folder, segs []string) (unloadedCategory, bool) { g := root for i, seg := range segs { switch n := g.children[seg].(type) { @@ -61,27 +61,26 @@ func unloadedAt(root *folder, segs []string) (pendingLoad, bool) { continue case nil: if _, unloaded := g.unloaded[seg]; unloaded { - return pendingLoad{dir: segs[:i:i], in: g, name: seg}, true + return unloadedCategory{dir: segs[:i:i], in: g, name: seg}, true } } - return pendingLoad{}, false + return unloadedCategory{}, false } - return pendingLoad{}, false + return unloadedCategory{}, false } // loadShippedAt loads the shipped category a caller's path names or descends into, unless // it is loaded already. -func (f *Generator) loadShippedAt(segs []string) error { - if p, unloaded := unloadedAt(&f.root, segs); unloaded { - return loadShipped(&f.root, []pendingLoad{p}) +func (f *Generator) loadShippedAt(segs []string) { + if u, unloaded := unloadedAt(&f.root, segs); unloaded { + loadShipped(&f.root, []unloadedCategory{u}) } - return nil } -// shippedReads is every unloaded shipped category the templates of scope reference, each +// unloadedReads is every unloaded shipped category the templates of scope reference, each // reference read from the folder dir. -func shippedReads(root *folder, dir []string, scope nodeScope) []pendingLoad { - var out []pendingLoad +func unloadedReads(root *folder, dir []string, scope nodeScope) []unloadedCategory { + var out []unloadedCategory _ = scope(func(_ string, n node) error { t, isTemplate := n.(*template) if !isTemplate { @@ -92,8 +91,8 @@ func shippedReads(root *folder, dir []string, scope nodeScope) []pendingLoad { if err != nil { continue // the link reports it } - if p, unloaded := unloadedAt(root, segs); unloaded { - out = append(out, p) + if u, unloaded := unloadedAt(root, segs); unloaded { + out = append(out, u) } } return nil @@ -102,66 +101,59 @@ func shippedReads(root *folder, dir []string, scope nodeScope) []pendingLoad { } // loadShipped loads the shipped categories wanted, with every category they read and the -// whole table family of each, then binds what it loaded, as New binds a whole load. A failed -// bind leaves every category it loaded unloaded again. -func loadShipped(root *folder, wanted []pendingLoad) (err error) { - var loaded []pendingLoad - var entries []shippedCategory - defer func() { - if err == nil { - return - } - for i, p := range loaded { - delete(p.in.children, p.name) - p.in.unloaded[p.name] = entries[i] - } - }() +// whole table family of each, then binds what it loaded, as New binds a whole load. +// TestEveryShippedCategoryLoadsAlone proves each closure, so a failure is the index's. +func loadShipped(root *folder, wanted []unloadedCategory) { var sites []categorySite for queue := wanted; len(queue) > 0; queue = queue[1:] { - p := queue[0] - c, unloaded := p.in.unloaded[p.name] + u := queue[0] + e, unloaded := u.in.unloaded[u.name] if !unloaded { continue } - site, err := p.load() + site, err := u.load() if err != nil { - return err + panic(internalError("shipped %s: %v", u.name, err)) } - loaded, entries = append(loaded, p), append(entries, c) sites = append(sites, site) - queue = append(queue, shippedReads(root, p.dir, sitesScope([]categorySite{site}))...) - queue = append(queue, p.family(c)...) + queue = append(queue, unloadedReads(root, u.dir, sitesScope([]categorySite{site}))...) + queue = append(queue, u.family(e)...) + } + if len(sites) == 0 { + return } sort.Slice(sites, func(i, j int) bool { return sites[i].path < sites[j].path }) - return categoryBinding(sites, root.children).bind() + if err := categoryBinding(sites, root.children).bind(); err != nil { + panic(internalError("binding shipped categories: %v", err)) + } } // load parses and compiles the category, and moves it from its folder's unloaded // categories to its children. -func (p pendingLoad) load() (categorySite, error) { - dir := path.Join(append([]string{shippedSource.baseDir}, p.dir...)...) +func (u unloadedCategory) load() (categorySite, error) { + dir := path.Join(append([]string{shippedSource.baseDir}, u.dir...)...) entries, err := fs.ReadDir(shippedSource.fsys, dir) if err != nil { return categorySite{}, fmt.Errorf("%s: %w", dir, err) } - file := p.name + ".json" - if err := loadFile(shippedSource, p.in, path.Join(dir, file), file, newCategoryFiles(shippedSource, dir, entries)); err != nil { + file := u.name + ".json" + if err := loadFile(shippedSource, u.in, path.Join(dir, file), file, newCategoryFiles(shippedSource, dir, entries)); err != nil { return categorySite{}, err } - delete(p.in.unloaded, p.name) - return categorySite{dir: p.dir, path: join(strings.Join(p.dir, "."), p.name), in: p.in, n: p.in.children[p.name]}, nil + delete(u.in.unloaded, u.name) + return categorySite{dir: u.dir, path: join(strings.Join(u.dir, "."), u.name), in: u.in, n: u.in.children[u.name]}, nil } -// family is the tables beside p that a table family links it to: its parent, and every +// family is the tables beside u that a table family links it to: its parent, and every // table whose parent it is. -func (p pendingLoad) family(c shippedCategory) []pendingLoad { - var out []pendingLoad - if c.parent != "" { - out = append(out, pendingLoad{dir: p.dir, in: p.in, name: c.parent}) +func (u unloadedCategory) family(e shippedEntry) []unloadedCategory { + var out []unloadedCategory + if e.parent != "" { + out = append(out, unloadedCategory{dir: u.dir, in: u.in, name: e.parent}) } - for name, sibling := range p.in.unloaded { - if sibling.parent == p.name { - out = append(out, pendingLoad{dir: p.dir, in: p.in, name: name}) + for name, sibling := range u.in.unloaded { + if sibling.parent == u.name { + out = append(out, unloadedCategory{dir: u.dir, in: u.in, name: name}) } } return out diff --git a/shippedindex.go b/shippedindex.go index e9fb3a7..6e73c43 100644 --- a/shippedindex.go +++ b/shippedindex.go @@ -2,7 +2,7 @@ package fejkdata -var shippedIndex = map[string]shippedCategory{ +var shippedIndex = map[string]shippedEntry{ "en_US.address": {paths: []string{"", "locality", "postal-code", "region", "street", "street-number"}}, "en_US.color": {paths: []string{""}}, "en_US.company": {paths: []string{"", "base", "suffix"}}, diff --git a/struct.go b/struct.go index 0ee799a..0f74bf7 100644 --- a/struct.go +++ b/struct.go @@ -258,7 +258,8 @@ func (s *structShape) compileRecord(root *folder, t reflect.Type, label string, if err != nil { return fmt.Errorf("%s: %w", label, err) } - b := inlineBinding(n, label, root) + loadShipped(root, unloadedReads(root, nil, inlineScope(n, label))) + b := inlineBinding(n, label, root.children) b.typedByGo = true if err := b.bind(); err != nil { return err