From 55d503317dc7d2e35f5d8300538aff6a722afd72 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sat, 10 Oct 2026 11:07:16 +0200 Subject: [PATCH] Answer architecture round 3: New loads only sources without an index and what they read, an entry drops reads, ErrLoad sits beside New, the manifest decoder moves to manifest.go and names each mistake once as jsonvalue.Kind words it --- AGENTS.md | 2 +- CHANGELOG.md | 4 +- README.md | 14 +-- data.go | 58 ++------- data/.fejkdata.json | 18 +-- datatype.go | 3 +- docs/decisions.md | 41 ++++--- fejkdata.go | 13 ++- index.go | 67 ----------- internal/datafiles/datafiles.go | 177 ---------------------------- internal/datafiles/manifest.go | 200 ++++++++++++++++++++++++++++++++ internal/jsonvalue/jsonvalue.go | 20 ++++ node.go | 22 +--- perf_test.go | 2 +- shipped_test.go | 2 +- table.go | 3 +- 16 files changed, 294 insertions(+), 352 deletions(-) create mode 100644 internal/datafiles/manifest.go diff --git a/AGENTS.md b/AGENTS.md index 1e98851..0e3dace 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,7 +47,7 @@ In [docs/decisions.md](docs/decisions.md): - `FakeTemplate` and `NewTemplate` both stay - The shipped data is embedded, not discovered - The shipped data is Go packages a library imports by choice, and the CLI carries every one, from 0.1.0 -- A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads and every indexed category depending on what it provides or a later source replaced +- A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads - Every `{…}` draws afresh, and only a name keeps a pick - Reference sigils follow the filesystem - A change to what exists is a major; a minor only adds diff --git a/CHANGELOG.md b/CHANGELOG.md index ca16767..7793a5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -128,5 +128,5 @@ replacement, and each removed path, column or flag. - A data source whose root holds a `.fejkdata.json` manifest with an `index` loads each category on the first call reaching it, as the shipped set does. Data failing to load, in `New` or there, gives an error matching `ErrLoad`, and the CLI exits 1. A source without - one loads in `New`, with the shipped categories it reads and every shipped category - depending on one it replaces; the rest of the shipped set still loads on first reach. + one loads in `New`, with the shipped categories it reads; the rest of the shipped set, + a reader of a category it replaces too, loads on first reach. diff --git a/README.md b/README.md index 5451179..f5fe925 100644 --- a/README.md +++ b/README.md @@ -174,13 +174,13 @@ a category may name a hidden rows file beside it. A source whose root holds `.fejkdata.json` with an `index` loads each category on the first call reaching it, and its author proves each loads. Any other source loads in -`New`, and so does every indexed category that depends on what such a source provides -or on a category a later source replaced. The index names every category of its source, -so a category file it leaves out never loads. The shipped set's -[`data/.fejkdata.json`](data/.fejkdata.json) shows the shape: each category's dot path, -its `parent` table, an entry of the same index, every path `List` advertises below it, -`""` for the category itself, and the other categories its templates `reads`. A table -whose parent column differs from its entry's `parent` fails at first reach. +`New`, with the indexed categories it reads. A shipped category reading one that another +source replaced loads on its own first reach, and fails there if the replacement breaks +it. The index names every category of its source, so a category file it leaves out never +loads. The shipped set's [`data/.fejkdata.json`](data/.fejkdata.json) shows the shape: +each category's dot path, its `parent` table, an entry of the same index, and every path +`List` advertises below it, `""` for the category itself. A table whose parent column +differs from its entry's `parent` fails at first reach. `//go:embed` of a directory leaves the manifest out; name it in the pattern too. Each locale carries `address`, `color`, `company`, `date`, `email`, `first-name`, diff --git a/data.go b/data.go index 919c334..68dbc53 100644 --- a/data.go +++ b/data.go @@ -7,67 +7,33 @@ import ( ) // loadSources merges sources in order, the last winning a clash. A source whose manifest -// carries an index leaves its categories unloaded; every other source loads here. So do -// the unloaded categories that depend on one it loaded or on one a later source replaced, -// so that a reader such a change breaks fails New, not a first reach. +// carries an index leaves its categories unloaded; every other source loads here, with +// the unloaded categories it needs. func loadSources(sources []datafiles.Source) (folder, error) { root := folder{children: map[string]node{}} - var changed []string for _, src := range sources { - replaced, err := addSource(&root, src) - if err != nil { + if err := addSource(&root, src); err != nil { return folder{}, err } - changed = append(changed, replaced...) } - sites := categorySites(&root) - for _, s := range sites { - changed = append(changed, s.path) - } - return root, loadReached(&root, sites, affectedBy(&root, changed)) + return root, loadReached(&root, categorySites(&root), nil) } -// addSource adds the categories of src to root, and returns the path of each category -// already there that src replaced or put out of reach. -func addSource(root *folder, src datafiles.Source) ([]string, error) { +func addSource(root *folder, src datafiles.Source) error { m, err := src.Manifest() if err != nil { - return nil, err + return err } - before := standing(root) if m.Index != nil { placeIndex(root, src, m.Index) - } else { - g := &folder{children: map[string]node{}} - if err := src.Walk(compileInto(func(dir []string) *folder { return madeFolder(g, dir) })); err != nil { - return nil, err - } - mergeFolder(root, g) + return nil } - if len(before) == 0 { - return nil, nil + g := &folder{children: map[string]node{}} + if err := src.Walk(compileInto(func(dir []string) *folder { return madeFolder(g, dir) })); err != nil { + return err } - after := standing(root) - var replaced []string - for p, was := range before { - if after[p] != was { - replaced = append(replaced, p) - } - } - return replaced, nil -} - -// standing is what stands at the path of every category under root: its node, or its -// unloaded entry. -func standing(root *folder) map[string]any { - out := map[string]any{} - for _, s := range categorySites(root) { - out[s.path] = s.n - } - for p, c := range unloadedUnder(root) { - out[p] = c.in.unloaded[c.name] - } - return out + mergeFolder(root, g) + return nil } func compileInto(place func(dir []string) *folder) func(datafiles.Category) error { diff --git a/data/.fejkdata.json b/data/.fejkdata.json index a778f75..150a4a9 100644 --- a/data/.fejkdata.json +++ b/data/.fejkdata.json @@ -1,6 +1,6 @@ { "index": { - "en_US.address": {"paths":["","locality","postal-code","region","street","street-number"],"reads":["geo.US.address"]}, + "en_US.address": {"paths":["","locality","postal-code","region","street","street-number"]}, "en_US.color": {"paths":[""]}, "en_US.company": {"paths":["","base","suffix"]}, "en_US.date": {"paths":[""]}, @@ -9,8 +9,8 @@ "en_US.ip": {"paths":[""]}, "en_US.itin": {"paths":["","area","group","serial"]}, "en_US.last-name": {"paths":["","count","name"]}, - "en_US.person": {"paths":["","first","last","prefix","sex"],"reads":["en_US.first-name","en_US.last-name"]}, - "en_US.phone": {"paths":["","area","exchange","line"],"reads":["en_US.phone-exchange"]}, + "en_US.person": {"paths":["","first","last","prefix","sex"]}, + "en_US.phone": {"paths":["","area","exchange","line"]}, "en_US.phone-exchange": {"paths":[""]}, "en_US.price": {"paths":["","amt","cents"]}, "en_US.sentence": {"paths":["","adj","noun","prep","verb"]}, @@ -22,13 +22,13 @@ "en_US.username": {"paths":["","n"]}, "en_US.version": {"paths":["","n","pre","suffix"]}, "en_US.word": {"paths":[""]}, - "geo.SE.address": {"paths":["","locality","postal-code","street","street-number"],"reads":["geo.SE.locality"]}, + "geo.SE.address": {"paths":["","locality","postal-code","street","street-number"]}, "geo.SE.locality": {"parent":"municipality","paths":["","lat","lon","municipality","name","population","postal-code","postal-code.code","postal-code.locality","street","street.locality","street.name","street.segments"]}, "geo.SE.municipality": {"parent":"region","paths":["","code","locality","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.locality","locality.street.name","locality.street.segments","name","population","region"]}, "geo.SE.postal-code": {"parent":"locality","paths":["","code","locality"]}, "geo.SE.region": {"paths":["","code","municipality","municipality.code","municipality.locality","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.locality","municipality.locality.street.name","municipality.locality.street.segments","municipality.name","municipality.population","municipality.region","name","population","timezone"]}, "geo.SE.street": {"parent":"locality","paths":["","locality","name","segments"]}, - "geo.US.address": {"paths":["","locality","postal-code","region","street","street-number"],"reads":["geo.US.locality"]}, + "geo.US.address": {"paths":["","locality","postal-code","region","street","street-number"]}, "geo.US.locality": {"parent":"municipality","paths":["","code","lat","lon","municipality","name","population","postal-code","postal-code.addresses","postal-code.code","postal-code.locality","street","street.addresses","street.locality","street.name"]}, "geo.US.municipality": {"parent":"region","paths":["","code","locality","locality.code","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.addresses","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.addresses","locality.street.locality","locality.street.name","name","population","region"]}, "geo.US.postal-code": {"parent":"locality","paths":["","addresses","code","locality"]}, @@ -54,7 +54,7 @@ "misc.tld": {"paths":["","tld","type","unicode"]}, "misc.useragent": {"paths":["","browser","device","os","ua"]}, "misc.uuid": {"paths":["","variant"]}, - "sv_SE.address": {"paths":["","locality","postal-code","street","street-number"],"reads":["geo.SE.address"]}, + "sv_SE.address": {"paths":["","locality","postal-code","street","street-number"]}, "sv_SE.birth-number": {"parent":"sex","paths":["","number","sex"]}, "sv_SE.color": {"paths":[""]}, "sv_SE.company": {"paths":["","base","suffix"]}, @@ -63,11 +63,11 @@ "sv_SE.first-name": {"parent":"sex","paths":["","count","name","sex"]}, "sv_SE.ip": {"paths":[""]}, "sv_SE.last-name": {"paths":["","count","name"]}, - "sv_SE.person": {"paths":["","first","last","prefix","sex"],"reads":["sv_SE.first-name","sv_SE.last-name","sv_SE.title"]}, - "sv_SE.personnummer": {"paths":[""],"reads":["sv_SE.birth-number"]}, + "sv_SE.person": {"paths":["","first","last","prefix","sex"]}, + "sv_SE.personnummer": {"paths":[""]}, "sv_SE.phone": {"paths":["","a","b","c","prefix"]}, "sv_SE.price": {"paths":["","amt","ore"]}, - "sv_SE.samordningsnummer": {"paths":[""],"reads":["sv_SE.birth-number"]}, + "sv_SE.samordningsnummer": {"paths":[""]}, "sv_SE.sentence": {"paths":["","adj","noun","prep","verb"]}, "sv_SE.sex": {"paths":["","birth-number","birth-number.number","birth-number.sex","code","first-name","first-name.count","first-name.name","first-name.sex","name"]}, "sv_SE.time": {"paths":[""]}, diff --git a/datatype.go b/datatype.go index 42939cc..03f18b5 100644 --- a/datatype.go +++ b/datatype.go @@ -7,6 +7,7 @@ import ( "github.com/larvit/fejkdata/internal/datatype" "github.com/larvit/fejkdata/internal/grammar" "github.com/larvit/fejkdata/internal/invariant" + "github.com/larvit/fejkdata/internal/jsonvalue" ) // DataType is what a record column holds, which decides how a record writes its value. It @@ -29,7 +30,7 @@ func datatypeOf(m map[string]any, pos position) (*DataType, error) { } name, ok := v.(string) if !ok { - return nil, fmt.Errorf("datatype must be a string, not %s", jsonKind(v)) + return nil, fmt.Errorf("datatype must be a string, not %s", jsonvalue.Kind(v)) } for d := DataTypeString; d < datatype.Count; d++ { if name != d.String() { diff --git a/docs/decisions.md b/docs/decisions.md index 7e67314..24f9dfd 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -154,29 +154,32 @@ still lives in `data/`; `--data-path` layers over it. Valid while the shipped data stays under 100 MiB, which a gate test holds (`todo.md` item 192): `go get` fetches the whole Go module, and the `go` command refuses a Go module zip over 500 MiB (`MaxZipFile` in `golang.org/x/mod/zip`). Past it, `todo.md` item 193 moves the largest packages into nested Go modules that keep their import paths, each requiring the core release that drops the package, so no build sees it twice. -## A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads and every indexed category depending on what it provides or a later source replaced +## A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads -2026-10-03, a restructure the maintainer approved; widened to any source on 2026-10-09. +2026-10-03, a restructure the maintainer approved; widened to any source on 2026-10-09, +and narrowed by the maintainer on 2026-10-10. Goal 13: `New` pays nothing for a category a run never reaches. A manifest, -`.fejkdata.json` at a source's root, may carry an index: each category's parent table, -the paths `List` advertises below it and the categories its templates reference. A -category of an indexed source loads with every category it references and its whole -table family, and binds as a whole load does, so it renders the same as after a whole -load. `go generate` writes the shipped set's manifest from `data/`; -`TestShippedManifestIsCurrent` fails while it is stale. CI loads the whole shipped set, -and each shipped category alone into a fresh `New`, so none fails at first reach. +`.fejkdata.json` at a source's root, may carry an index: each category's parent table and +the paths `List` advertises below it. A category of an indexed source loads with every +category it references and its whole table family, and binds as a whole load does, so +it renders the same as after a whole load. `go generate` writes the shipped set's +manifest from `data/`; `TestShippedManifestIsCurrent` fails while it is stale. CI loads +the whole shipped set, and each shipped category alone into a fresh `New`, so none fails +at first reach. -Goal 6: a source without an index loads whole in `New`, so each mistake in it is `New`'s -error. So does every indexed category it reads, and every indexed category depending, -through references or table links, on a category it provides or a later source -replaced, so a replacement or a stand-in breaking a reader fails `New`. Goal 6.3: an -indexed source from anyone else may fail at first reach; `Fake` then returns the load's -error, matching `ErrLoad`, the same every call. A first reach checks only a table's -`parent` against its entry, so a table loads with its parent and a failed load puts -back whole. Proving the rest of an index is its author's, with the tool -`todo.md` item 194 plans: checked against the merged tree, an entry's paths and reads -would blame the manifest for a table or a stand-in another source adds. +Goal 6: a source without an index loads whole in `New`, with every indexed category it +reads, so each mistake in it is `New`'s error. Goal 6.3: an indexed source from anyone +else may fail at first reach; the call then returns the load's error, matching +`ErrLoad`, the same every call. So does an indexed category reading one that another +source replaced, where the replacement breaks it. The maintainer chose that on +2026-10-10 over loading in `New` every indexed category depending on a replaced one: +readers of three rounds ruled that rule harder to follow (goal 3), and it needed every +entry to list what its templates read. + +A first reach checks only a table's `parent` against its entry, so a table loads with its +parent and a failed load puts back whole. Proving the rest of an index is its author's, +with the tool `todo.md` item 194 plans. Valid while CI proves the shipped set whole and each category alone. diff --git a/fejkdata.go b/fejkdata.go index b312d6f..3da4e72 100644 --- a/fejkdata.go +++ b/fejkdata.go @@ -3,6 +3,7 @@ package fejkdata import ( crand "crypto/rand" "encoding/binary" + "errors" "fmt" "io/fs" "reflect" @@ -67,6 +68,16 @@ func WithoutShippedData() Option { return func(c *config) { c.shipped = false } } +// ErrLoad marks data that fails to load: in New, or on the first call reaching a category +// of an indexed source. +var ErrLoad = errors.New("data fails to load") + +// loadError is a load's error, unchanged in its text, matching ErrLoad. +type loadError struct{ error } + +func (e loadError) Unwrap() error { return e.error } +func (loadError) Is(target error) bool { return target == ErrLoad } + // New builds a generator from the shipped data set and the options' sources, merged // in order with the last winning a name clash. Each JSON file becomes a category // named after the file (address.json -> "address") and each subdirectory a @@ -91,7 +102,7 @@ func New(opts ...Option) (*Generator, error) { return &Generator{drawState: drawstate.New(seed), root: root}, nil } -// load builds the tree New starts from. docs/decisions.md#a-source-whose-manifest-carries-an-index-loads-each-category-on-first-reach-any-other-loads-in-new-with-what-it-reads-and-every-indexed-category-depending-on-what-it-provides-or-a-later-source-replaced +// load builds the tree New starts from. docs/decisions.md#a-source-whose-manifest-carries-an-index-loads-each-category-on-first-reach-any-other-loads-in-new-with-what-it-reads func (c config) load() (folder, error) { var sources []datafiles.Source if c.shipped { diff --git a/index.go b/index.go index 04e6de9..c3803ad 100644 --- a/index.go +++ b/index.go @@ -1,26 +1,14 @@ package fejkdata import ( - "errors" "fmt" "slices" - "sort" "strings" "github.com/larvit/fejkdata/internal/datafiles" "github.com/larvit/fejkdata/internal/grammar" ) -// ErrLoad marks data that fails to load: in New, or on the first call reaching a category -// of an indexed source. -var ErrLoad = errors.New("data fails to load") - -// loadError is a load's error, unchanged in its text, matching ErrLoad. -type loadError struct{ error } - -func (e loadError) Unwrap() error { return e.error } -func (loadError) Is(target error) bool { return target == ErrLoad } - // indexed is a category an index names and no call has reached yet, and the source holding it. type indexed struct { entry datafiles.IndexEntry @@ -113,61 +101,6 @@ func needs(root *folder, s categorySite) []categoryAt { return out } -// affectedBy is every unloaded category under root whose index entry leads to one of -// paths: through its reads, its parent or a child table, and through other unloaded -// categories. -func affectedBy(root *folder, paths []string) []categoryAt { - if len(paths) == 0 { - return nil - } - unloaded := unloadedUnder(root) - readers := map[string][]string{} - for _, p := range sortedNames(unloaded) { - c := unloaded[p] - e := c.in.unloaded[c.name].entry - for _, r := range e.Reads { - readers[r] = append(readers[r], p) - } - if e.Parent != "" { - parent := categoryPath(c.dir, e.Parent) - readers[parent] = append(readers[parent], p) - readers[p] = append(readers[p], parent) - } - } - var out []categoryAt - seen := map[string]bool{} - queue := slices.Clone(paths) - sort.Strings(queue) - for ; len(queue) > 0; queue = queue[1:] { - for _, p := range readers[queue[0]] { - if c, isUnloaded := unloaded[p]; isUnloaded && !seen[p] { - seen[p] = true - out = append(out, c) - queue = append(queue, p) - } - } - } - return out -} - -func unloadedUnder(root *folder) map[string]categoryAt { - out := map[string]categoryAt{} - var walk func(g *folder, dir []string) - walk = func(g *folder, dir []string) { - for name := range g.unloaded { - c := categoryAt{dir: dir, in: g, name: name} - out[categoryPath(dir, name)] = c - } - for name, n := range g.children { - if sub, isFolder := n.(*folder); isFolder { - walk(sub, append(dir[:len(dir):len(dir)], name)) - } - } - } - walk(root, nil) - return out -} - // loadReached loads the unloaded categories among wanted and every unloaded category they // or sites need, then runs sites and all it loaded through a whole load's pipeline. On an // error it puts back what it loaded, so the next call fails the same way. diff --git a/internal/datafiles/datafiles.go b/internal/datafiles/datafiles.go index e63c9e1..5739bc7 100644 --- a/internal/datafiles/datafiles.go +++ b/internal/datafiles/datafiles.go @@ -2,13 +2,10 @@ package datafiles import ( - "encoding/json" - "errors" "fmt" "io/fs" "os" "path" - "sort" "strings" "github.com/larvit/fejkdata/internal/grammar" @@ -59,180 +56,6 @@ func (s Source) dirPath(dir []string) string { return "." } -// ManifestFile is the file at a source's root that describes the source. -const ManifestFile = ".fejkdata.json" - -// Manifest is what a source's manifest says of it. -type Manifest struct { - // Index is every category of the source by dot path; nil where the manifest has none. - Index map[string]IndexEntry `json:"index"` -} - -// IndexEntry is what an index says of a category: its parent table, "" for none, the paths -// List advertises below it, and the categories its templates reference. -type IndexEntry struct { - Parent string `json:"parent,omitempty"` - Paths []string `json:"paths"` - Reads []string `json:"reads,omitempty"` -} - -// Manifest reads the manifest at the source's root, the zero Manifest where there is none. -func (s Source) Manifest() (Manifest, error) { - if err := s.check(); err != nil { - return Manifest{}, err - } - b, err := fs.ReadFile(s.fsys, path.Join(s.base, ManifestFile)) - if errors.Is(err, fs.ErrNotExist) { - return Manifest{}, nil - } - if err != nil { - return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err) - } - m, err := decodeManifest(b) - if err != nil { - return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err) - } - return m, nil -} - -// ManifestPath is the manifest's path as errors name it. -func (s Source) ManifestPath() string { return s.labelled(path.Join(s.base, ManifestFile)) } - -// decodeManifest decodes a manifest, refusing any key or shape it does not define, and -// reports every mistake in the order of the entries naming them. -func decodeManifest(b []byte) (Manifest, error) { - var raw any - if err := json.Unmarshal(b, &raw); err != nil { - return Manifest{}, err - } - top, isObject := raw.(map[string]any) - if !isObject { - return Manifest{}, fmt.Errorf("a manifest is a JSON object") - } - var m Manifest - var errs []error - for _, k := range sortedKeys(top) { - if k != "index" { - errs = append(errs, fmt.Errorf("unknown key %q; a manifest holds index", k)) - continue - } - index, isObject := top[k].(map[string]any) - if !isObject { - errs = append(errs, fmt.Errorf("index is a JSON object of category paths")) - continue - } - m.Index = map[string]IndexEntry{} - for _, p := range sortedKeys(index) { - e, err := decodeEntry(p, index[p]) - if err != nil { - errs = append(errs, fmt.Errorf("index entry %q: %w", p, err)) - } - m.Index[p] = e - } - } - for _, p := range sortedKeys(m.Index) { - if err := checkParent(m.Index, p); err != nil { - errs = append(errs, fmt.Errorf("index entry %q: %w", p, err)) - } - } - if len(errs) > 0 { - return Manifest{}, errors.Join(errs...) - } - return m, nil -} - -func decodeEntry(p string, v any) (IndexEntry, error) { - var e IndexEntry - errs := []error{checkCategoryPath(p)} - fields, isObject := v.(map[string]any) - if !isObject { - return e, fmt.Errorf("an entry is a JSON object of parent, paths and reads") - } - if _, has := fields["paths"]; !has { - errs = append(errs, fmt.Errorf("paths is missing")) - } - var err error - for _, k := range sortedKeys(fields) { - switch k { - case "parent": - var isString bool - if e.Parent, isString = fields[k].(string); !isString { - err = fmt.Errorf("parent is a string") - } else if err = grammar.CheckIdentifier(e.Parent); err != nil { - err = fmt.Errorf("parent %w", err) - } - case "paths": - e.Paths, err = stringList(k, fields[k], func(q string) error { - if q == "" { - return nil - } - return checkCategoryPath(q) - }) - case "reads": - e.Reads, err = stringList(k, fields[k], checkCategoryPath) - default: - err = fmt.Errorf("unknown key %q; an entry holds parent, paths and reads", k) - } - errs = append(errs, err) - } - return e, errors.Join(errs...) -} - -// stringList is the JSON array of strings v, each passing check. -func stringList(key string, v any, check func(string) error) ([]string, error) { - items, isArray := v.([]any) - if !isArray { - return nil, fmt.Errorf("%s is an array of strings", key) - } - out := make([]string, 0, len(items)) - for _, item := range items { - s, isString := item.(string) - if !isString { - return nil, fmt.Errorf("%s is an array of strings", key) - } - if err := check(s); err != nil { - return nil, fmt.Errorf("%s: %w", key, err) - } - out = append(out, s) - } - return out, nil -} - -// checkParent refuses a parent naming no entry beside p: a table loads with its parent, -// from one source. -func checkParent(index map[string]IndexEntry, p string) error { - parent := index[p].Parent - if parent == "" { - return nil - } - sibling := parent - if dot := strings.LastIndexByte(p, '.'); dot >= 0 { - sibling = p[:dot+1] + parent - } - if _, ok := index[sibling]; !ok { - return fmt.Errorf("parent %q names no entry beside it in this index", parent) - } - return nil -} - -func checkCategoryPath(p string) error { - for _, seg := range strings.Split(p, ".") { - if err := grammar.CheckIdentifier(seg); err != nil { - return fmt.Errorf("%q: %w", p, err) - } - } - return nil -} - -func sortedKeys[V any](m map[string]V) []string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - sort.Strings(keys) - return keys -} - // Walk hands compile every category of the tree, folders and files in name order. A // folder holding no category anywhere below it is skipped; a hidden file or folder is no // category, though a category may name a hidden rows file beside it. diff --git a/internal/datafiles/manifest.go b/internal/datafiles/manifest.go new file mode 100644 index 0000000..25f1a2a --- /dev/null +++ b/internal/datafiles/manifest.go @@ -0,0 +1,200 @@ +package datafiles + +import ( + "errors" + "fmt" + "io/fs" + "path" + "sort" + "strings" + + "github.com/larvit/fejkdata/internal/grammar" + "github.com/larvit/fejkdata/internal/jsonvalue" +) + +// ManifestFile is the file at a source's root that describes the source. +const ManifestFile = ".fejkdata.json" + +// Manifest is what a source's manifest says of it. +type Manifest struct { + // Index is every category of the source by dot path; nil where the manifest has none. + Index map[string]IndexEntry `json:"index"` +} + +// IndexEntry is what an index says of a category: its parent table, "" for none, and the +// paths List advertises below it. +type IndexEntry struct { + Parent string `json:"parent,omitempty"` + Paths []string `json:"paths"` +} + +// Manifest reads the manifest at the source's root, the zero Manifest where there is none. +func (s Source) Manifest() (Manifest, error) { + if err := s.check(); err != nil { + return Manifest{}, err + } + b, err := fs.ReadFile(s.fsys, path.Join(s.base, ManifestFile)) + if errors.Is(err, fs.ErrNotExist) { + return Manifest{}, nil + } + if err != nil { + return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err) + } + m, err := decodeManifest(b) + if err != nil { + return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err) + } + return m, nil +} + +// ManifestPath is the manifest's path as errors name it. +func (s Source) ManifestPath() string { return s.labelled(path.Join(s.base, ManifestFile)) } + +// decodeManifest decodes a manifest, refusing any key or shape it does not define, and +// reports every mistake: first each entry's own, in path order, then each parent naming +// no entry. +func decodeManifest(b []byte) (Manifest, error) { + raw, err := jsonvalue.Decode(b) + if err != nil { + return Manifest{}, err + } + top, isObject := raw.Any().(map[string]any) + if !isObject { + return Manifest{}, fmt.Errorf("a manifest must be an object, not %s", jsonvalue.Kind(raw.Any())) + } + var m Manifest + var errs []error + for _, k := range sortedKeys(top) { + if k != "index" { + errs = append(errs, fmt.Errorf("unknown key %q; a manifest holds index", k)) + continue + } + var indexErrs []error + m.Index, indexErrs = decodeIndex(top[k]) + errs = append(errs, indexErrs...) + } + return m, errors.Join(errs...) +} + +func decodeIndex(v any) (map[string]IndexEntry, []error) { + entries, isObject := v.(map[string]any) + if !isObject { + return nil, []error{fmt.Errorf("index must be an object, not %s", jsonvalue.Kind(v))} + } + index := map[string]IndexEntry{} + var errs []error + for _, p := range sortedKeys(entries) { + e, err := decodeEntry(p, entries[p]) + if err != nil { + errs = append(errs, fmt.Errorf("index entry %q: %w", p, err)) + } + index[p] = e + } + for _, p := range sortedKeys(index) { + if err := checkParent(index, p); err != nil { + errs = append(errs, fmt.Errorf("index entry %q: %w", p, err)) + } + } + return index, errs +} + +// decodeEntry decodes one entry, leaving out a parent it refuses, so checkParent names no +// mistake twice. +func decodeEntry(p string, v any) (IndexEntry, error) { + var e IndexEntry + errs := []error{checkCategoryPath(p)} + fields, isObject := v.(map[string]any) + if !isObject { + errs = append(errs, fmt.Errorf("an entry must be an object, not %s", jsonvalue.Kind(v))) + return e, errors.Join(errs...) + } + if _, has := fields["paths"]; !has { + errs = append(errs, fmt.Errorf("paths is missing")) + } + for _, k := range sortedKeys(fields) { + switch k { + case "parent": + parent, err := decodeParent(fields[k]) + e.Parent = parent + errs = append(errs, err) + case "paths": + var err error + e.Paths, err = decodePaths(fields[k]) + errs = append(errs, err) + default: + errs = append(errs, fmt.Errorf("unknown key %q; an entry holds parent and paths", k)) + } + } + return e, errors.Join(errs...) +} + +func decodeParent(v any) (string, error) { + parent, isString := v.(string) + if !isString { + return "", fmt.Errorf("parent must be a string, not %s", jsonvalue.Kind(v)) + } + if err := grammar.CheckIdentifier(parent); err != nil { + return "", fmt.Errorf("parent %w", err) + } + return parent, nil +} + +// decodePaths decodes a list of paths below a category, "" being the category itself. +func decodePaths(v any) ([]string, error) { + items, isList := v.([]any) + if !isList { + return nil, fmt.Errorf("paths must be a list, not %s", jsonvalue.Kind(v)) + } + out := make([]string, 0, len(items)) + var errs []error + for i, item := range items { + q, isString := item.(string) + if !isString { + errs = append(errs, fmt.Errorf("paths item %d must be a string, not %s", i+1, jsonvalue.Kind(item))) + continue + } + if q != "" { + if err := checkCategoryPath(q); err != nil { + errs = append(errs, fmt.Errorf("paths item %d: %w", i+1, err)) + continue + } + } + out = append(out, q) + } + return out, errors.Join(errs...) +} + +// checkParent refuses a parent naming no entry beside p: a table loads with its parent, +// from one source. +func checkParent(index map[string]IndexEntry, p string) error { + parent := index[p].Parent + if parent == "" { + return nil + } + sibling := parent + if dot := strings.LastIndexByte(p, '.'); dot >= 0 { + sibling = p[:dot+1] + parent + } + if _, ok := index[sibling]; !ok { + return fmt.Errorf("parent %q names no entry beside it in this index", parent) + } + return nil +} + +func checkCategoryPath(p string) error { + for _, seg := range strings.Split(p, ".") { + if err := grammar.CheckIdentifier(seg); err != nil { + return fmt.Errorf("%q: %w", p, err) + } + } + return nil +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/internal/jsonvalue/jsonvalue.go b/internal/jsonvalue/jsonvalue.go index c477f51..17ba339 100644 --- a/internal/jsonvalue/jsonvalue.go +++ b/internal/jsonvalue/jsonvalue.go @@ -4,6 +4,7 @@ package jsonvalue import ( "bytes" "encoding/json" + "fmt" ) // Value is a JSON value that only Decode, String and Object make, so a template never compiles @@ -37,3 +38,22 @@ func Object(fields map[string]Value) Value { // Any is the value as encoding/json decodes it: a map[string]any, []any, string, json.Number, // bool or nil. func (v Value) Any() any { return v.v } + +// Kind names the kind of a value as Any gives it, such as "a number" or "a list". +func Kind(v any) string { + switch v.(type) { + case json.Number: + return "a number" + case bool: + return "a boolean" + case string: + return "a string" + case []any: + return "a list" + case map[string]any: + return "an object" + case nil: + return "null" + } + return fmt.Sprintf("%T", v) +} diff --git a/node.go b/node.go index b25035b..dc9eeda 100644 --- a/node.go +++ b/node.go @@ -193,22 +193,6 @@ func typedOrText(text string, ds ...DataType) error { // jsonKind names the kind of a JSON value an option cannot take, in the data format's own // terms rather than the decoding library's. -func jsonKind(v any) string { - switch v.(type) { - case json.Number: - return "a number" - case bool: - return "a boolean" - case string: - return "a string" - case []any: - return "a list" - case map[string]any: - return "an object" - } - return fmt.Sprintf("%T", v) -} - func compileString(s string) (*template, error) { toks, unbound, err := parseChecked(s, nil) if err != nil { @@ -332,7 +316,7 @@ func readOptions(m map[string]any, pos position) (templateOptions, error) { } if sv, ok := m["separator"]; ok { if o.separator, ok = sv.(string); !ok { - return o, fmt.Errorf("separator must be a string, not %s", jsonKind(sv)) + return o, fmt.Errorf("separator must be a string, not %s", jsonvalue.Kind(sv)) } } return o, nil @@ -365,7 +349,7 @@ func repeatOf(m map[string]any) (int, error) { } r, ok := numberOf(rv) if !ok { - return 0, fmt.Errorf("repeat must be a number, not %s", jsonKind(rv)) + return 0, fmt.Errorf("repeat must be a number, not %s", jsonvalue.Kind(rv)) } if err := notTiny("repeat", rv, r); err != nil { return 0, err @@ -392,7 +376,7 @@ func weightOf(raw any) (float64, error) { } w, ok := numberOf(wv) if !ok { - return 0, fmt.Errorf(`weight is an option and takes a number, not %s; a field cannot be named "weight"`, jsonKind(wv)) + return 0, fmt.Errorf(`weight is an option and takes a number, not %s; a field cannot be named "weight"`, jsonvalue.Kind(wv)) } if w < 0 || math.IsInf(w, 0) { return 0, fmt.Errorf("weight must be a finite number of 0 or more, got %v", wv) diff --git a/perf_test.go b/perf_test.go index 996caae..dfd95b0 100644 --- a/perf_test.go +++ b/perf_test.go @@ -177,7 +177,7 @@ func BenchmarkWideTokens500(b *testing.B) { // New decodes the shipped manifest and loads nothing else. func TestNoNewAllocRegression(t *testing.T) { - const base = 2501.0 + const base = 2449.0 allocs := testing.AllocsPerRun(100, func() { if _, err := New(WithSeed(1)); err != nil { t.Fatal(err) diff --git a/shipped_test.go b/shipped_test.go index bba4ad0..1b21c1b 100644 --- a/shipped_test.go +++ b/shipped_test.go @@ -263,7 +263,7 @@ func TestABrokenManifestFailsNewNamingEveryMistake(t *testing.T) { `{`: {datafiles.ManifestFile}, `{"index": {}} {}`: {"after top-level value"}, `{"indx": {}, "INDEX": {}}`: {`unknown key "INDEX"`, `unknown key "indx"`}, - `{"index": []}`: {"index must be an object, not an array"}, + `{"index": []}`: {"index must be an object, not a list"}, `{"index": {"a..b": 5}}`: {`"a..b"`, "must be an object, not a number"}, `{"index": {"a": {"Paths": [""]}}}`: {`unknown key "Paths"`, "paths is missing"}, `{"index": {"a": {}, "b": {}}}`: {`"a": paths is missing`, `"b": paths is missing`}, diff --git a/table.go b/table.go index d02ddb8..5221135 100644 --- a/table.go +++ b/table.go @@ -7,6 +7,7 @@ import ( "github.com/larvit/fejkdata/internal/drawstate" "github.com/larvit/fejkdata/internal/grammar" "github.com/larvit/fejkdata/internal/invariant" + "github.com/larvit/fejkdata/internal/jsonvalue" "github.com/larvit/fejkdata/internal/rows" ) @@ -102,7 +103,7 @@ func readTableOptions(m map[string]any) (tableOptionValues, error) { } s, isString := v.(string) if !isString { - return o, fmt.Errorf("%s must be a string, not %s", k, jsonKind(v)) + return o, fmt.Errorf("%s must be a string, not %s", k, jsonvalue.Kind(v)) } if k != "format" && s == "" { return o, fmt.Errorf("%s names a column, so it cannot be empty; drop it", k)