larvit/fejkdata
3
0
Fork 0
Code Pull Requests 1 Actions Activity

Load shipped categories before binding, panic on an index that misses a read, and name the index entry and unloaded category plainly

Browse Source
This commit is contained in:
Lilleman auf Larv
2026-10-03 17:20:36 +02:00
committed by Lillem4n
parent 81bb979164
commit 8b3bbc4003
10 changed files with 70 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AGENTS.md
+1 -1
View File
@@ -43,7 +43,7 @@ In [docs/decisions.md](docs/decisions.md):
- A padded JSON argument is rejected, not trimmed
- `FakeTemplate` and `NewTemplate` both stay
- The shipped data is embedded, not discovered
- The shipped set alone loads a category on the first call reaching it; beside a `--data-path` it loads whole
- The shipped set alone loads a category on the first call reaching it; beside a `--data-path` every category loads in `New`
- A bare reference draws each time; a reference path is held
- Reference sigils follow the filesystem
- A change to what exists is a major; a minor only adds
docs/decisions.md
+4 -4
View File
@@ -116,17 +116,17 @@ still lives in `data/` as JSON; `--data-path` layers over it.
`todo.md` item 77 revises it: a library imports the shipped data as Go modules.
## The shipped set alone loads a category on the first call reaching it; beside a `--data-path` it loads whole
## The shipped set alone loads a category on the first call reaching it; beside a `--data-path` every category loads in `New`
2026-10-03, approved by the maintainer as the restructure of `todo.md` item 76. Goal 12: a
`New` pays nothing for the categories a run never reaches, which matters once the shipped
set grows to hundreds of MB. A category loads with every category it references and its
whole table family, and is bound as a whole load binds, so it renders as it would from
`New`. `List` reads `shippedindex.go`, generated from `data/` with `REPIN=1`, and
`TestShippedIndexIsCurrent` fails while it is stale. CI loads the whole shipped set, so a
shipped category never fails at first reach. Goal 6: beside a `--data-path` or a
`TestShippedIndexIsCurrent` fails while it is stale. CI loads the whole shipped set, and
loads each shipped category alone into a fresh `New`, so none fails at first reach. Goal 6: beside a `--data-path` or a
`WithDataFS`, every category loads in `New`, so each mistake in a source the user wrote is
`New`'s error. Valid while the shipped set is proven whole in CI.
`New`'s error. Valid while CI proves the shipped set whole and each category alone.
## A bare reference draws each time; a reference path is held
fejkdata.go
+3 -3
View File
@@ -108,7 +108,7 @@ func New(opts ...Option) (*Generator, error) {
// mistake in that source is New's error.
func (c config) load() (folder, error) {
if c.shipped && len(c.sources) == 0 {
return shelve(), nil
return unloadedTree(), nil
}
var sources []dataSource
if c.shipped {
@@ -144,8 +144,8 @@ func paths(n node) []string {
out = append(out, join(name, p))
}
}
for name, c := range n.unloaded {
for _, p := range c.paths {
for name, e := range n.unloaded {
for _, p := range e.paths {
out = append(out, join(name, p))
}
}
inline.go
+4 -8
View File
@@ -34,7 +34,8 @@ func (f *Generator) NewTemplate(input string) (*Template, error) {
}
f.mu.Lock()
defer f.mu.Unlock()
if err := inlineBinding(n, "template", &f.root).bind(); err != nil {
loadShipped(&f.root, unloadedReads(&f.root, nil, inlineScope(n, "template")))
if err := inlineBinding(n, "template", f.root.children).bind(); err != nil {
return nil, fmt.Errorf("fejkdata: %w", err)
}
return &Template{g: f, n: n}, nil
@@ -148,16 +149,11 @@ func inputValue(input string) (any, error) {
return raw, nil
}
func inlineBinding(n node, label string, root *folder) binding {
func inlineBinding(n node, label string, root map[string]node) binding {
scope := inlineScope(n, label)
return binding{
scope: scope,
link: func() error {
if err := loadShipped(root, shippedReads(root, nil, scope)); err != nil {
return err
}
return linkNodeRefs(scope, root.children)
},
link: func() error { return linkNodeRefs(scope, root) },
scopeFence: func() error {
if t, isTemplate := n.(*template); isTemplate && t.drawGroup != "" {
return fmt.Errorf("%s: drawGroup %q names nothing, since nothing can reference an inline template; drop it", label, t.drawGroup)
node.go
+1 -1
View File
@@ -18,7 +18,7 @@ type node interface{ isNode() }
// dot path; rendering one is an error (see Fake).
type folder struct {
children map[string]node
unloaded map[string]shippedCategory // the shipped categories here a call has not reached yet
unloaded map[string]shippedEntry // the shipped categories here a call has not reached yet
}
func (*folder) isNode() {}
record.go
+1 -3
View File
@@ -126,9 +126,7 @@ func (f *Generator) FakeRecord(path string) (*Record, error) {
if err != nil {
return nil, fmt.Errorf("fejkdata: %w", err)
}
if err := f.loadShippedAt(segments); err != nil {
return nil, fmt.Errorf("fejkdata: %w", err)
}
f.loadShippedAt(segments)
_, n, tail, err := resolveCategory(f.root.children, segments)
if err != nil {
return nil, fmt.Errorf("fejkdata: %s: %w", path, err)
render.go
+1 -3
View File
@@ -24,9 +24,7 @@ func (f *Generator) Fake(path string) (string, error) {
if err != nil {
return "", fmt.Errorf("fejkdata: %w", err)
}
if err := f.loadShippedAt(segments); err != nil {
return "", fmt.Errorf("fejkdata: %w", err)
}
f.loadShippedAt(segments)
var draws renderDraws
sc := renderScope{draws: &draws}
n, err := descend(f.rand, &f.root, segments, sc)
shipped.go
+52 -60
View File
@@ -14,17 +14,17 @@ var shippedFS embed.FS
var shippedSource = dataSource{fsys: shippedFS, baseDir: "data"}
// shippedCategory is a shipped category as shippedindex.go lists it: the table beside it
// that it links to, "" for none, and the paths List advertises below it.
type shippedCategory struct {
// shippedEntry is a shipped category's entry in shippedindex.go: the table beside it that
// it links to, "" for none, and the paths List advertises below it.
type shippedEntry struct {
parent string
paths []string
}
// shelve builds the folders of the shipped index, with every category in them unloaded.
func shelve() folder {
// unloadedTree builds the folders of the shipped index, with every category in them unloaded.
func unloadedTree() folder {
root := folder{children: map[string]node{}}
for p, c := range shippedIndex {
for p, e := range shippedIndex {
segs := strings.Split(p, ".")
g := &root
for _, seg := range segs[:len(segs)-1] {
@@ -36,23 +36,23 @@ func shelve() folder {
g = sub
}
if g.unloaded == nil {
g.unloaded = map[string]shippedCategory{}
g.unloaded = map[string]shippedEntry{}
}
g.unloaded[segs[len(segs)-1]] = c
g.unloaded[segs[len(segs)-1]] = e
}
return root
}
// pendingLoad is an unloaded shipped category: the folder holding it, that folder's path,
// and its name there.
type pendingLoad struct {
// unloadedCategory is a shipped category not loaded yet: the folder holding it, that
// folder's path, and its name there.
type unloadedCategory struct {
dir []string
in *folder
name string
}
// unloadedAt is the unloaded shipped category a path from root names, or descends into.
func unloadedAt(root *folder, segs []string) (pendingLoad, bool) {
func unloadedAt(root *folder, segs []string) (unloadedCategory, bool) {
g := root
for i, seg := range segs {
switch n := g.children[seg].(type) {
@@ -61,27 +61,26 @@ func unloadedAt(root *folder, segs []string) (pendingLoad, bool) {
continue
case nil:
if _, unloaded := g.unloaded[seg]; unloaded {
return pendingLoad{dir: segs[:i:i], in: g, name: seg}, true
return unloadedCategory{dir: segs[:i:i], in: g, name: seg}, true
}
}
return pendingLoad{}, false
return unloadedCategory{}, false
}
return pendingLoad{}, false
return unloadedCategory{}, false
}
// loadShippedAt loads the shipped category a caller's path names or descends into, unless
// it is loaded already.
func (f *Generator) loadShippedAt(segs []string) error {
if p, unloaded := unloadedAt(&f.root, segs); unloaded {
return loadShipped(&f.root, []pendingLoad{p})
func (f *Generator) loadShippedAt(segs []string) {
if u, unloaded := unloadedAt(&f.root, segs); unloaded {
loadShipped(&f.root, []unloadedCategory{u})
}
return nil
}
// shippedReads is every unloaded shipped category the templates of scope reference, each
// unloadedReads is every unloaded shipped category the templates of scope reference, each
// reference read from the folder dir.
func shippedReads(root *folder, dir []string, scope nodeScope) []pendingLoad {
var out []pendingLoad
func unloadedReads(root *folder, dir []string, scope nodeScope) []unloadedCategory {
var out []unloadedCategory
_ = scope(func(_ string, n node) error {
t, isTemplate := n.(*template)
if !isTemplate {
@@ -92,8 +91,8 @@ func shippedReads(root *folder, dir []string, scope nodeScope) []pendingLoad {
if err != nil {
continue // the link reports it
}
if p, unloaded := unloadedAt(root, segs); unloaded {
out = append(out, p)
if u, unloaded := unloadedAt(root, segs); unloaded {
out = append(out, u)
}
}
return nil
@@ -102,66 +101,59 @@ func shippedReads(root *folder, dir []string, scope nodeScope) []pendingLoad {
}
// loadShipped loads the shipped categories wanted, with every category they read and the
// whole table family of each, then binds what it loaded, as New binds a whole load. A failed
// bind leaves every category it loaded unloaded again.
func loadShipped(root *folder, wanted []pendingLoad) (err error) {
var loaded []pendingLoad
var entries []shippedCategory
defer func() {
if err == nil {
return
}
for i, p := range loaded {
delete(p.in.children, p.name)
p.in.unloaded[p.name] = entries[i]
}
}()
// whole table family of each, then binds what it loaded, as New binds a whole load.
// TestEveryShippedCategoryLoadsAlone proves each closure, so a failure is the index's.
func loadShipped(root *folder, wanted []unloadedCategory) {
var sites []categorySite
for queue := wanted; len(queue) > 0; queue = queue[1:] {
p := queue[0]
c, unloaded := p.in.unloaded[p.name]
u := queue[0]
e, unloaded := u.in.unloaded[u.name]
if !unloaded {
continue
}
site, err := p.load()
site, err := u.load()
if err != nil {
return err
panic(internalError("shipped %s: %v", u.name, err))
}
loaded, entries = append(loaded, p), append(entries, c)
sites = append(sites, site)
queue = append(queue, shippedReads(root, p.dir, sitesScope([]categorySite{site}))...)
queue = append(queue, p.family(c)...)
queue = append(queue, unloadedReads(root, u.dir, sitesScope([]categorySite{site}))...)
queue = append(queue, u.family(e)...)
}
if len(sites) == 0 {
return
}
sort.Slice(sites, func(i, j int) bool { return sites[i].path < sites[j].path })
return categoryBinding(sites, root.children).bind()
if err := categoryBinding(sites, root.children).bind(); err != nil {
panic(internalError("binding shipped categories: %v", err))
}
}
// load parses and compiles the category, and moves it from its folder's unloaded
// categories to its children.
func (p pendingLoad) load() (categorySite, error) {
dir := path.Join(append([]string{shippedSource.baseDir}, p.dir...)...)
func (u unloadedCategory) load() (categorySite, error) {
dir := path.Join(append([]string{shippedSource.baseDir}, u.dir...)...)
entries, err := fs.ReadDir(shippedSource.fsys, dir)
if err != nil {
return categorySite{}, fmt.Errorf("%s: %w", dir, err)
}
file := p.name + ".json"
if err := loadFile(shippedSource, p.in, path.Join(dir, file), file, newCategoryFiles(shippedSource, dir, entries)); err != nil {
file := u.name + ".json"
if err := loadFile(shippedSource, u.in, path.Join(dir, file), file, newCategoryFiles(shippedSource, dir, entries)); err != nil {
return categorySite{}, err
}
delete(p.in.unloaded, p.name)
return categorySite{dir: p.dir, path: join(strings.Join(p.dir, "."), p.name), in: p.in, n: p.in.children[p.name]}, nil
delete(u.in.unloaded, u.name)
return categorySite{dir: u.dir, path: join(strings.Join(u.dir, "."), u.name), in: u.in, n: u.in.children[u.name]}, nil
}
// family is the tables beside p that a table family links it to: its parent, and every
// family is the tables beside u that a table family links it to: its parent, and every
// table whose parent it is.
func (p pendingLoad) family(c shippedCategory) []pendingLoad {
var out []pendingLoad
if c.parent != "" {
out = append(out, pendingLoad{dir: p.dir, in: p.in, name: c.parent})
func (u unloadedCategory) family(e shippedEntry) []unloadedCategory {
var out []unloadedCategory
if e.parent != "" {
out = append(out, unloadedCategory{dir: u.dir, in: u.in, name: e.parent})
}
for name, sibling := range p.in.unloaded {
if sibling.parent == p.name {
out = append(out, pendingLoad{dir: p.dir, in: p.in, name: name})
for name, sibling := range u.in.unloaded {
if sibling.parent == u.name {
out = append(out, unloadedCategory{dir: u.dir, in: u.in, name: name})
}
}
return out
shippedindex.go
+1 -1
View File
@@ -2,7 +2,7 @@
package fejkdata
var shippedIndex = map[string]shippedCategory{
var shippedIndex = map[string]shippedEntry{
"en_US.address": {paths: []string{"", "locality", "postal-code", "region", "street", "street-number"}},
"en_US.color": {paths: []string{""}},
"en_US.company": {paths: []string{"", "base", "suffix"}},
struct.go
+2 -1
View File
@@ -258,7 +258,8 @@ func (s *structShape) compileRecord(root *folder, t reflect.Type, label string,
if err != nil {
return fmt.Errorf("%s: %w", label, err)
}
b := inlineBinding(n, label, root)
loadShipped(root, unloadedReads(root, nil, inlineScope(n, label)))
b := inlineBinding(n, label, root.children)
b.typedByGo = true
if err := b.bind(); err != nil {
return err