From 498f81075e3dc86d09da73f1225b6f45cef651e0 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 9 Oct 2026 19:36:27 +0200 Subject: [PATCH] Answer architecture round 2: a first reach checks only a table's parent, loadReached splits into a batch that loads, runs and puts back, the manifest decodes strictly and reports every mistake, and ErrLoad marks a load failing in New too --- CHANGELOG.md | 4 +- README.md | 12 +-- data.go | 11 +-- docs/comprehension-history.md | 22 +++++ docs/decisions.md | 7 +- fejkdata.go | 5 +- index.go | 169 +++++++++++++++----------------- inline.go | 2 +- internal/datafiles/datafiles.go | 138 +++++++++++++++++++++----- 9 files changed, 235 insertions(+), 135 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e677fd4..ca16767 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -126,7 +126,7 @@ replacement, and each removed path, column or flag. `sv_SE.sex[f].first-name[Kim]`, and a category with no columns names the record that gives it one. - A data source whose root holds a `.fejkdata.json` manifest with an `index` loads each - category on the first call reaching it, as the shipped set does, and a category failing - there returns an error wrapping `ErrLoad`, on which the CLI exits 1. A source without + category on the first call reaching it, as the shipped set does. Data failing to load, + in `New` or there, gives an error matching `ErrLoad`, and the CLI exits 1. A source without one loads in `New`, with the shipped categories it reads and every shipped category depending on one it replaces; the rest of the shipped set still loads on first reach. diff --git a/README.md b/README.md index 9607294..5451179 100644 --- a/README.md +++ b/README.md @@ -178,10 +178,10 @@ first call reaching it, and its author proves each loads. Any other source loads or on a category a later source replaced. The index names every category of its source, so a category file it leaves out never loads. The shipped set's [`data/.fejkdata.json`](data/.fejkdata.json) shows the shape: each category's dot path, -its `parent` table, every path `List` advertises below it, `""` for the category itself, -and the other categories its templates `reads`. A category whose entry says otherwise -fails at first reach, naming the entry it calls for. `//go:embed` of a directory leaves -the manifest out; name it in the pattern too. +its `parent` table, an entry of the same index, every path `List` advertises below it, +`""` for the category itself, and the other categories its templates `reads`. A table +whose parent column differs from its entry's `parent` fails at first reach. +`//go:embed` of a directory leaves the manifest out; name it in the pattern too. Each locale carries `address`, `color`, `company`, `date`, `email`, `first-name`, `ip`, `last-name`, `person`, `phone`, `price`, `sentence`, `sex`, `time`, `url`, @@ -342,8 +342,8 @@ ok, err := fejkdata.IsTemplate(arg) // an inline template by its shape, el `New` refuses data that breaks the grammar, could mean two things or cannot render a valid value, and `NewTemplate` refuses such a template. A category of an indexed source is -checked on the first call reaching it instead, and a failure there wraps `ErrLoad`, the -same error every call. On a loaded generator: +checked on the first call reaching it instead. Data that fails to load, in `New` or +there, gives an error matching `ErrLoad`, the same every call. On a loaded generator: - `Fake` fails only for a path that names nothing, could name two things, or reads one draw of a level carrying a `repeat`, with the same error every call, or with `ErrLoad`. diff --git a/data.go b/data.go index 7351834..919c334 100644 --- a/data.go +++ b/data.go @@ -24,7 +24,7 @@ func loadSources(sources []datafiles.Source) (folder, error) { for _, s := range sites { changed = append(changed, s.path) } - return root, loadReached(&root, sites, dependents(&root, changed)) + return root, loadReached(&root, sites, affectedBy(&root, changed)) } // addSource adds the categories of src to root, and returns the path of each category @@ -111,13 +111,10 @@ func mergeFolder(dst, src *folder) { // single inline node. type nodeSet func(fn func(label string, n node) error) error -// categorySite is a loaded category and where it sits: the folder holding it, that -// folder's path, its name and its own dot path. +// categorySite is a loaded category, where it sits and its own dot path. type categorySite struct { - dir []string - in *folder + categoryAt n node - name string path string } @@ -139,7 +136,7 @@ func categorySites(root *folder) []categorySite { } func siteIn(dir []string, in *folder, name string) categorySite { - return categorySite{dir: dir, in: in, n: in.children[name], name: name, path: categoryPath(dir, name)} + return categorySite{categoryAt: categoryAt{dir: dir, in: in, name: name}, n: in.children[name], path: categoryPath(dir, name)} } func categoryPath(dir []string, name string) string { return join(strings.Join(dir, "."), name) } diff --git a/docs/comprehension-history.md b/docs/comprehension-history.md index 9a3d38d..8c780d1 100644 --- a/docs/comprehension-history.md +++ b/docs/comprehension-history.md @@ -202,3 +202,25 @@ Inherited architect, decided by: - Shape: `index.go:159` `reaching`: four near-synonyms, `reaching`, `reachedFrom`, `loadReached` and `referenced`, and `referenced` and `categoryUnder` return loaded categories too - Shape: `data.go:40` `place`: shares its name with `compileInto`'s parameter, and returns a closure answering one question two ways - Locality: `internal/datafiles/datafiles.go:83` `embeddedManifests`: hidden global state for a small saving + +## 2026-10-09T17:32:44Z, PR #202 at fdc6a04, against 2652779 + +Ruling: worse + +| Seat | Navigation | Locality | Shape | Self-sufficiency | Overall | +|---|---|---|---|---|---| +| Mid A | same | worse | same | same | worse | +| Inherited architect | same | worse | worse | worse | worse | + +Mid A, decided by: + +- Locality, Overall: `index.go:175` `loadReached`: one loop loads, puts back on failure, checks each entry before linking and again after the pipeline, and its rollback rests on an ordering invariant its comment argues +- Locality: `data.go:13` `loadSources`: whether a category loads in New means holding every source and every index entry together, through `standing` snapshots and `dependents` +- Shape: `index.go:36` `categoryAt`: a near-synonym of `categorySite`, beside `dependencies` and `dependents`, which differ by two letters and run in opposite directions + +Inherited architect, decided by: + +- Locality, Overall: `index.go:175` `loadReached`: five jobs in one queue loop, with a cross-call ordering invariant to hold +- Locality, Shape: `data.go:13` `loadSources`: one source can make another load early through identity-compared snapshots and a reverse graph over index entries +- Shape: `index.go:36` `categoryAt`: two near-synonym types for where a category sits, and three near-identical function names +- Self-sufficiency: `index.go:248` `staleEntry`: its note on linking cannot be read without knowing the pipeline, and `config.load` points at a paragraph-long decision anchor diff --git a/docs/decisions.md b/docs/decisions.md index 608f2d8..918d9d7 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -172,8 +172,11 @@ error. So does every indexed category it reads, and every indexed category depen through references or table links, on a category it provides or a later source replaced, so a replacement or a stand-in breaking a reader fails `New`. Goal 6.3: an indexed source from anyone else may fail at first reach; `Fake` then returns the load's -error wrapping `ErrLoad`, the same every call. A category whose index entry disagrees -with it fails there too, so a stale index never decides what loads. +error, matching `ErrLoad`, the same every call. A first reach checks only a table's +`parent` against its entry, so a table loads with its parent and a failed load puts +back whole. Proving the rest of an index is its author's, with the tool `todo.md` item +194 plans: checked against the merged tree, an entry's paths and reads would blame the +manifest for a table or a stand-in another source adds. Valid while CI proves the shipped set whole and each category alone. diff --git a/fejkdata.go b/fejkdata.go index c749b59..b312d6f 100644 --- a/fejkdata.go +++ b/fejkdata.go @@ -73,7 +73,8 @@ func WithoutShippedData() Option { // namespace segment. It errors on a missing directory, invalid JSON or invalid data; a // generator with no data at all renders templates that read none. A source whose // .fejkdata.json carries an index, as the shipped set's does, loads each category on the -// first call reaching it, and fails there with [ErrLoad]. +// first call reaching it. Data failing to load, here or there, gives an error matching +// [ErrLoad]. func New(opts ...Option) (*Generator, error) { c := config{shipped: true} for _, opt := range opts { @@ -81,7 +82,7 @@ func New(opts ...Option) (*Generator, error) { } root, err := c.load() if err != nil { - return nil, fmt.Errorf("fejkdata: %w", err) + return nil, fmt.Errorf("fejkdata: %w", loadError{err}) } seed, err := c.drawSeed() if err != nil { diff --git a/index.go b/index.go index 1456d4e..04e6de9 100644 --- a/index.go +++ b/index.go @@ -1,7 +1,6 @@ package fejkdata import ( - "encoding/json" "errors" "fmt" "slices" @@ -12,10 +11,16 @@ import ( "github.com/larvit/fejkdata/internal/grammar" ) -// ErrLoad marks the error of a category that fails to load on the first call reaching it, -// which only a source whose manifest carries an index defers to then. +// ErrLoad marks data that fails to load: in New, or on the first call reaching a category +// of an indexed source. var ErrLoad = errors.New("data fails to load") +// loadError is a load's error, unchanged in its text, matching ErrLoad. +type loadError struct{ error } + +func (e loadError) Unwrap() error { return e.error } +func (loadError) Is(target error) bool { return target == ErrLoad } + // indexed is a category an index names and no call has reached yet, and the source holding it. type indexed struct { entry datafiles.IndexEntry @@ -39,7 +44,7 @@ type categoryAt struct { name string } -func (c categoryAt) path() string { return categoryPath(c.dir, c.name) } +func (c categoryAt) unloaded() bool { return c.in.unloaded[c.name] != nil } // categoryOn is the category, loaded or not, that a path from root names or passes. func categoryOn(root *folder, segs []string) (categoryAt, bool) { @@ -47,10 +52,8 @@ func categoryOn(root *folder, segs []string) (categoryAt, bool) { if i == len(segs) { return categoryAt{}, false } - if _, unloaded := g.unloaded[segs[i]]; !unloaded && g.children[segs[i]] == nil { - return categoryAt{}, false - } - return categoryAt{dir: segs[:i:i], in: g, name: segs[i]}, true + c := categoryAt{dir: segs[:i:i], in: g, name: segs[i]} + return c, c.unloaded() || g.children[c.name] != nil } // loadCallerPath strips the / a caller's path may start with, splits the path, and loads the @@ -64,9 +67,9 @@ func (f *Generator) loadCallerPath(path string) (string, []string, error) { if err != nil { return "", nil, fmt.Errorf("fejkdata: %w", err) } - if c, ok := categoryOn(&f.root, segs); ok { + if c, ok := categoryOn(&f.root, segs); ok && c.unloaded() { if err := loadReached(&f.root, nil, []categoryAt{c}); err != nil { - return "", nil, fmt.Errorf("fejkdata: %w: %w", ErrLoad, err) + return "", nil, fmt.Errorf("fejkdata: %w", loadError{err}) } } return path, segs, nil @@ -95,9 +98,9 @@ func referenced(root *folder, dir []string, nodes nodeSet) []categoryAt { return out } -// dependencies is every category, loaded or not, that s references, the table beside s -// that its parent column names, and every unloaded table beside s whose parent s is. -func dependencies(root *folder, s categorySite) []categoryAt { +// needs is every category, loaded or not, that s references, the table beside s that its +// parent column names, and every unloaded table beside s whose parent s is. +func needs(root *folder, s categorySite) []categoryAt { out := referenced(root, s.dir, siteNodes([]categorySite{s})) if parent := parentOf(s.n); parent != "" { out = append(out, categoryAt{dir: s.dir, in: s.in, name: parent}) @@ -110,25 +113,25 @@ func dependencies(root *folder, s categorySite) []categoryAt { return out } -// dependents is every unloaded category under root whose index entry leads to one of +// affectedBy is every unloaded category under root whose index entry leads to one of // paths: through its reads, its parent or a child table, and through other unloaded // categories. -func dependents(root *folder, paths []string) []categoryAt { +func affectedBy(root *folder, paths []string) []categoryAt { if len(paths) == 0 { return nil } unloaded := unloadedUnder(root) - from := map[string][]string{} // a path, and the unloaded categories leading to it + readers := map[string][]string{} for _, p := range sortedNames(unloaded) { c := unloaded[p] e := c.in.unloaded[c.name].entry for _, r := range e.Reads { - from[r] = append(from[r], p) + readers[r] = append(readers[r], p) } if e.Parent != "" { parent := categoryPath(c.dir, e.Parent) - from[parent] = append(from[parent], p) - from[p] = append(from[p], parent) + readers[parent] = append(readers[parent], p) + readers[p] = append(readers[p], parent) } } var out []categoryAt @@ -136,14 +139,12 @@ func dependents(root *folder, paths []string) []categoryAt { queue := slices.Clone(paths) sort.Strings(queue) for ; len(queue) > 0; queue = queue[1:] { - for _, p := range from[queue[0]] { - c, isUnloaded := unloaded[p] - if !isUnloaded || seen[p] { - continue + for _, p := range readers[queue[0]] { + if c, isUnloaded := unloaded[p]; isUnloaded && !seen[p] { + seen[p] = true + out = append(out, c) + queue = append(queue, p) } - seen[p] = true - out = append(out, c) - queue = append(queue, p) } } return out @@ -155,7 +156,7 @@ func unloadedUnder(root *folder) map[string]categoryAt { walk = func(g *folder, dir []string) { for name := range g.unloaded { c := categoryAt{dir: dir, in: g, name: name} - out[c.path()] = c + out[categoryPath(dir, name)] = c } for name, n := range g.children { if sub, isFolder := n.(*folder); isFolder { @@ -167,87 +168,71 @@ func unloadedUnder(root *folder) map[string]categoryAt { return out } -// loadReached loads the unloaded categories among wanted, and every unloaded category they -// or sites depend on, then runs sites and all it loaded through a whole load's pipeline. -// On an error it puts back what it loaded, so the next call fails the same way. Checking -// each parent column against its entry before linking keeps that whole: a table then -// loads with its parent, so no table loaded earlier links to one put back. +// loadReached loads the unloaded categories among wanted and every unloaded category they +// or sites need, then runs sites and all it loaded through a whole load's pipeline. On an +// error it puts back what it loaded, so the next call fails the same way. func loadReached(root *folder, sites []categorySite, wanted []categoryAt) error { queue := wanted for _, s := range sites { - queue = append(queue, dependencies(root, s)...) + queue = append(queue, needs(root, s)...) } - type load struct { - site categorySite - was *indexed - } - var loads []load - putBack := func() { - for _, l := range loads { - l.site.in.putUnloaded(l.site.name, l.was) - } - } - for ; len(queue) > 0; queue = queue[1:] { - c := queue[0] - x, unloaded := c.in.unloaded[c.name] - if !unloaded { - continue - } - err := x.src.Load(c.dir, c.name, compileInto(func([]string) *folder { return c.in })) - if err == nil { - loads = append(loads, load{site: siteIn(c.dir, c.in, c.name), was: x}) - if parentOf(c.in.children[c.name]) != x.entry.Parent { - err = staleEntry(root, loads[len(loads)-1].site, x) - } - } - if err != nil { - putBack() - return err - } - queue = append(queue, dependencies(root, loads[len(loads)-1].site)...) - } - for _, l := range loads { - sites = append(sites[:len(sites):len(sites)], l.site) - } - if len(sites) == 0 { - return nil - } - err := categoryPipeline(sites, root.children).run() - for _, l := range loads { - if err == nil && !sameEntry(indexEntry(root, l.site), l.was.entry) { - err = staleEntry(root, l.site, l.was) - } + var b batch + err := b.load(root, queue) + if err == nil { + err = b.run(root, sites) } if err != nil { - putBack() + b.putBack() } return err } -// indexEntry is what an index says of the loaded category s. -func indexEntry(root *folder, s categorySite) datafiles.IndexEntry { - e := datafiles.IndexEntry{Parent: parentOf(s.n), Paths: paths(s.n, false)} - sort.Strings(e.Paths) - seen := map[string]bool{s.path: true} - for _, c := range referenced(root, s.dir, siteNodes([]categorySite{s})) { - if p := c.path(); !seen[p] { - seen[p] = true - e.Reads = append(e.Reads, p) +// batch is the categories one loadReached loaded, each beside the entry it was loaded from. +type batch []loaded + +type loaded struct { + site categorySite + was *indexed +} + +// load loads each unloaded category in queue and what it needs. A table whose parent +// column disagrees with its entry stops it, so a table loads with its parent and no table +// loaded before links to one putBack removes. +func (b *batch) load(root *folder, queue []categoryAt) error { + for ; len(queue) > 0; queue = queue[1:] { + c := queue[0] + x := c.in.unloaded[c.name] + if x == nil { + continue } + if err := x.src.Load(c.dir, c.name, compileInto(func([]string) *folder { return c.in })); err != nil { + return err + } + s := siteIn(c.dir, c.in, c.name) + *b = append(*b, loaded{site: s, was: x}) + if parent := parentOf(s.n); parent != x.entry.Parent { + return fmt.Errorf("%s: the index entry for %s names parent %q, and its table's parent column is %q", x.src.ManifestPath(), s.path, x.entry.Parent, parent) + } + queue = append(queue, needs(root, s)...) } - sort.Strings(e.Reads) - return e + return nil } -func sameEntry(a, b datafiles.IndexEntry) bool { - return a.Parent == b.Parent && slices.Equal(a.Paths, b.Paths) && slices.Equal(a.Reads, b.Reads) +func (b batch) run(root *folder, sites []categorySite) error { + sites = slices.Clip(sites) + for _, l := range b { + sites = append(sites, l.site) + } + if len(sites) == 0 { + return nil + } + return categoryPipeline(sites, root.children).run() } -// staleEntry names the entry x holds for s, and the entry s calls for. Before linking, -// the paths and reads it names may still lack what s's family adds. -func staleEntry(root *folder, s categorySite, x *indexed) error { - want, _ := json.Marshal(indexEntry(root, s)) - return fmt.Errorf("%s: the index entry for %s is stale; the category calls for %s", x.src.ManifestPath(), s.path, want) +func (b batch) putBack() { + for _, l := range b { + l.site.in.putUnloaded(l.site.name, l.was) + } } func parentOf(n node) string { diff --git a/inline.go b/inline.go index a4344c5..a9e15b3 100644 --- a/inline.go +++ b/inline.go @@ -91,7 +91,7 @@ func loadInline(root *folder, n node, label string, typedByGo bool) error { return err } if err := loadReached(root, nil, referenced(root, nil, nodes)); err != nil { - return fmt.Errorf("%w: %w", ErrLoad, err) + return loadError{err} } return pipeline{ nodes: nodes, diff --git a/internal/datafiles/datafiles.go b/internal/datafiles/datafiles.go index cb6148b..e63c9e1 100644 --- a/internal/datafiles/datafiles.go +++ b/internal/datafiles/datafiles.go @@ -2,14 +2,13 @@ package datafiles import ( - "bytes" "encoding/json" "errors" "fmt" - "io" "io/fs" "os" "path" + "sort" "strings" "github.com/larvit/fejkdata/internal/grammar" @@ -99,39 +98,123 @@ func (s Source) Manifest() (Manifest, error) { // ManifestPath is the manifest's path as errors name it. func (s Source) ManifestPath() string { return s.labelled(path.Join(s.base, ManifestFile)) } +// decodeManifest decodes a manifest, refusing any key or shape it does not define, and +// reports every mistake in the order of the entries naming them. func decodeManifest(b []byte) (Manifest, error) { - dec := json.NewDecoder(bytes.NewReader(b)) - dec.DisallowUnknownFields() - var m Manifest - if err := dec.Decode(&m); err != nil { + var raw any + if err := json.Unmarshal(b, &raw); err != nil { return Manifest{}, err } - if _, err := dec.Token(); err != io.EOF { - return Manifest{}, fmt.Errorf("more than one JSON value") + top, isObject := raw.(map[string]any) + if !isObject { + return Manifest{}, fmt.Errorf("a manifest is a JSON object") } - for p, e := range m.Index { - for _, q := range append([]string{p}, e.Reads...) { - if err := checkCategoryPath(q); err != nil { - return Manifest{}, fmt.Errorf("index entry %q: %w", p, err) - } - } - if e.Parent == "" { + var m Manifest + var errs []error + for _, k := range sortedKeys(top) { + if k != "index" { + errs = append(errs, fmt.Errorf("unknown key %q; a manifest holds index", k)) continue } - if err := grammar.CheckIdentifier(e.Parent); err != nil { - return Manifest{}, fmt.Errorf("index entry %q: parent %w", p, err) + index, isObject := top[k].(map[string]any) + if !isObject { + errs = append(errs, fmt.Errorf("index is a JSON object of category paths")) + continue } - sibling := e.Parent - if dot := strings.LastIndexByte(p, '.'); dot >= 0 { - sibling = p[:dot+1] + e.Parent - } - if _, ok := m.Index[sibling]; !ok { - return Manifest{}, fmt.Errorf("index entry %q: parent %q names no entry beside it", p, e.Parent) + m.Index = map[string]IndexEntry{} + for _, p := range sortedKeys(index) { + e, err := decodeEntry(p, index[p]) + if err != nil { + errs = append(errs, fmt.Errorf("index entry %q: %w", p, err)) + } + m.Index[p] = e } } + for _, p := range sortedKeys(m.Index) { + if err := checkParent(m.Index, p); err != nil { + errs = append(errs, fmt.Errorf("index entry %q: %w", p, err)) + } + } + if len(errs) > 0 { + return Manifest{}, errors.Join(errs...) + } return m, nil } +func decodeEntry(p string, v any) (IndexEntry, error) { + var e IndexEntry + errs := []error{checkCategoryPath(p)} + fields, isObject := v.(map[string]any) + if !isObject { + return e, fmt.Errorf("an entry is a JSON object of parent, paths and reads") + } + if _, has := fields["paths"]; !has { + errs = append(errs, fmt.Errorf("paths is missing")) + } + var err error + for _, k := range sortedKeys(fields) { + switch k { + case "parent": + var isString bool + if e.Parent, isString = fields[k].(string); !isString { + err = fmt.Errorf("parent is a string") + } else if err = grammar.CheckIdentifier(e.Parent); err != nil { + err = fmt.Errorf("parent %w", err) + } + case "paths": + e.Paths, err = stringList(k, fields[k], func(q string) error { + if q == "" { + return nil + } + return checkCategoryPath(q) + }) + case "reads": + e.Reads, err = stringList(k, fields[k], checkCategoryPath) + default: + err = fmt.Errorf("unknown key %q; an entry holds parent, paths and reads", k) + } + errs = append(errs, err) + } + return e, errors.Join(errs...) +} + +// stringList is the JSON array of strings v, each passing check. +func stringList(key string, v any, check func(string) error) ([]string, error) { + items, isArray := v.([]any) + if !isArray { + return nil, fmt.Errorf("%s is an array of strings", key) + } + out := make([]string, 0, len(items)) + for _, item := range items { + s, isString := item.(string) + if !isString { + return nil, fmt.Errorf("%s is an array of strings", key) + } + if err := check(s); err != nil { + return nil, fmt.Errorf("%s: %w", key, err) + } + out = append(out, s) + } + return out, nil +} + +// checkParent refuses a parent naming no entry beside p: a table loads with its parent, +// from one source. +func checkParent(index map[string]IndexEntry, p string) error { + parent := index[p].Parent + if parent == "" { + return nil + } + sibling := parent + if dot := strings.LastIndexByte(p, '.'); dot >= 0 { + sibling = p[:dot+1] + parent + } + if _, ok := index[sibling]; !ok { + return fmt.Errorf("parent %q names no entry beside it in this index", parent) + } + return nil +} + func checkCategoryPath(p string) error { for _, seg := range strings.Split(p, ".") { if err := grammar.CheckIdentifier(seg); err != nil { @@ -141,6 +224,15 @@ func checkCategoryPath(p string) error { return nil } +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + // Walk hands compile every category of the tree, folders and files in name order. A // folder holding no category anywhere below it is skipped; a hidden file or folder is no // category, though a category may name a hidden rows file beside it.