diff --git a/AGENTS.md b/AGENTS.md index 1e98851..0e3dace 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,7 +47,7 @@ In [docs/decisions.md](docs/decisions.md): - `FakeTemplate` and `NewTemplate` both stay - The shipped data is embedded, not discovered - The shipped data is Go packages a library imports by choice, and the CLI carries every one, from 0.1.0 -- A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads and every indexed category depending on what it provides or a later source replaced +- A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads - Every `{…}` draws afresh, and only a name keeps a pick - Reference sigils follow the filesystem - A change to what exists is a major; a minor only adds diff --git a/CHANGELOG.md b/CHANGELOG.md index ca16767..7793a5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -128,5 +128,5 @@ replacement, and each removed path, column or flag. - A data source whose root holds a `.fejkdata.json` manifest with an `index` loads each category on the first call reaching it, as the shipped set does. Data failing to load, in `New` or there, gives an error matching `ErrLoad`, and the CLI exits 1. A source without - one loads in `New`, with the shipped categories it reads and every shipped category - depending on one it replaces; the rest of the shipped set still loads on first reach. + one loads in `New`, with the shipped categories it reads; the rest of the shipped set, + a reader of a category it replaces too, loads on first reach. diff --git a/README.md b/README.md index 5451179..f5fe925 100644 --- a/README.md +++ b/README.md @@ -174,13 +174,13 @@ a category may name a hidden rows file beside it. A source whose root holds `.fejkdata.json` with an `index` loads each category on the first call reaching it, and its author proves each loads. Any other source loads in -`New`, and so does every indexed category that depends on what such a source provides -or on a category a later source replaced. The index names every category of its source, -so a category file it leaves out never loads. The shipped set's -[`data/.fejkdata.json`](data/.fejkdata.json) shows the shape: each category's dot path, -its `parent` table, an entry of the same index, every path `List` advertises below it, -`""` for the category itself, and the other categories its templates `reads`. A table -whose parent column differs from its entry's `parent` fails at first reach. +`New`, with the indexed categories it reads. A shipped category reading one that another +source replaced loads on its own first reach, and fails there if the replacement breaks +it. The index names every category of its source, so a category file it leaves out never +loads. The shipped set's [`data/.fejkdata.json`](data/.fejkdata.json) shows the shape: +each category's dot path, its `parent` table, an entry of the same index, and every path +`List` advertises below it, `""` for the category itself. A table whose parent column +differs from its entry's `parent` fails at first reach. `//go:embed` of a directory leaves the manifest out; name it in the pattern too. Each locale carries `address`, `color`, `company`, `date`, `email`, `first-name`, diff --git a/data.go b/data.go index 919c334..68dbc53 100644 --- a/data.go +++ b/data.go @@ -7,67 +7,33 @@ import ( ) // loadSources merges sources in order, the last winning a clash. A source whose manifest -// carries an index leaves its categories unloaded; every other source loads here. So do -// the unloaded categories that depend on one it loaded or on one a later source replaced, -// so that a reader such a change breaks fails New, not a first reach. +// carries an index leaves its categories unloaded; every other source loads here, with +// the unloaded categories it needs. func loadSources(sources []datafiles.Source) (folder, error) { root := folder{children: map[string]node{}} - var changed []string for _, src := range sources { - replaced, err := addSource(&root, src) - if err != nil { + if err := addSource(&root, src); err != nil { return folder{}, err } - changed = append(changed, replaced...) } - sites := categorySites(&root) - for _, s := range sites { - changed = append(changed, s.path) - } - return root, loadReached(&root, sites, affectedBy(&root, changed)) + return root, loadReached(&root, categorySites(&root), nil) } -// addSource adds the categories of src to root, and returns the path of each category -// already there that src replaced or put out of reach. -func addSource(root *folder, src datafiles.Source) ([]string, error) { +func addSource(root *folder, src datafiles.Source) error { m, err := src.Manifest() if err != nil { - return nil, err + return err } - before := standing(root) if m.Index != nil { placeIndex(root, src, m.Index) - } else { - g := &folder{children: map[string]node{}} - if err := src.Walk(compileInto(func(dir []string) *folder { return madeFolder(g, dir) })); err != nil { - return nil, err - } - mergeFolder(root, g) + return nil } - if len(before) == 0 { - return nil, nil + g := &folder{children: map[string]node{}} + if err := src.Walk(compileInto(func(dir []string) *folder { return madeFolder(g, dir) })); err != nil { + return err } - after := standing(root) - var replaced []string - for p, was := range before { - if after[p] != was { - replaced = append(replaced, p) - } - } - return replaced, nil -} - -// standing is what stands at the path of every category under root: its node, or its -// unloaded entry. -func standing(root *folder) map[string]any { - out := map[string]any{} - for _, s := range categorySites(root) { - out[s.path] = s.n - } - for p, c := range unloadedUnder(root) { - out[p] = c.in.unloaded[c.name] - } - return out + mergeFolder(root, g) + return nil } func compileInto(place func(dir []string) *folder) func(datafiles.Category) error { diff --git a/data/.fejkdata.json b/data/.fejkdata.json index a778f75..150a4a9 100644 --- a/data/.fejkdata.json +++ b/data/.fejkdata.json @@ -1,6 +1,6 @@ { "index": { - "en_US.address": {"paths":["","locality","postal-code","region","street","street-number"],"reads":["geo.US.address"]}, + "en_US.address": {"paths":["","locality","postal-code","region","street","street-number"]}, "en_US.color": {"paths":[""]}, "en_US.company": {"paths":["","base","suffix"]}, "en_US.date": {"paths":[""]}, @@ -9,8 +9,8 @@ "en_US.ip": {"paths":[""]}, "en_US.itin": {"paths":["","area","group","serial"]}, "en_US.last-name": {"paths":["","count","name"]}, - "en_US.person": {"paths":["","first","last","prefix","sex"],"reads":["en_US.first-name","en_US.last-name"]}, - "en_US.phone": {"paths":["","area","exchange","line"],"reads":["en_US.phone-exchange"]}, + "en_US.person": {"paths":["","first","last","prefix","sex"]}, + "en_US.phone": {"paths":["","area","exchange","line"]}, "en_US.phone-exchange": {"paths":[""]}, "en_US.price": {"paths":["","amt","cents"]}, "en_US.sentence": {"paths":["","adj","noun","prep","verb"]}, @@ -22,13 +22,13 @@ "en_US.username": {"paths":["","n"]}, "en_US.version": {"paths":["","n","pre","suffix"]}, "en_US.word": {"paths":[""]}, - "geo.SE.address": {"paths":["","locality","postal-code","street","street-number"],"reads":["geo.SE.locality"]}, + "geo.SE.address": {"paths":["","locality","postal-code","street","street-number"]}, "geo.SE.locality": {"parent":"municipality","paths":["","lat","lon","municipality","name","population","postal-code","postal-code.code","postal-code.locality","street","street.locality","street.name","street.segments"]}, "geo.SE.municipality": {"parent":"region","paths":["","code","locality","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.locality","locality.street.name","locality.street.segments","name","population","region"]}, "geo.SE.postal-code": {"parent":"locality","paths":["","code","locality"]}, "geo.SE.region": {"paths":["","code","municipality","municipality.code","municipality.locality","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.locality","municipality.locality.street.name","municipality.locality.street.segments","municipality.name","municipality.population","municipality.region","name","population","timezone"]}, "geo.SE.street": {"parent":"locality","paths":["","locality","name","segments"]}, - "geo.US.address": {"paths":["","locality","postal-code","region","street","street-number"],"reads":["geo.US.locality"]}, + "geo.US.address": {"paths":["","locality","postal-code","region","street","street-number"]}, "geo.US.locality": {"parent":"municipality","paths":["","code","lat","lon","municipality","name","population","postal-code","postal-code.addresses","postal-code.code","postal-code.locality","street","street.addresses","street.locality","street.name"]}, "geo.US.municipality": {"parent":"region","paths":["","code","locality","locality.code","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.addresses","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.addresses","locality.street.locality","locality.street.name","name","population","region"]}, "geo.US.postal-code": {"parent":"locality","paths":["","addresses","code","locality"]}, @@ -54,7 +54,7 @@ "misc.tld": {"paths":["","tld","type","unicode"]}, "misc.useragent": {"paths":["","browser","device","os","ua"]}, "misc.uuid": {"paths":["","variant"]}, - "sv_SE.address": {"paths":["","locality","postal-code","street","street-number"],"reads":["geo.SE.address"]}, + "sv_SE.address": {"paths":["","locality","postal-code","street","street-number"]}, "sv_SE.birth-number": {"parent":"sex","paths":["","number","sex"]}, "sv_SE.color": {"paths":[""]}, "sv_SE.company": {"paths":["","base","suffix"]}, @@ -63,11 +63,11 @@ "sv_SE.first-name": {"parent":"sex","paths":["","count","name","sex"]}, "sv_SE.ip": {"paths":[""]}, "sv_SE.last-name": {"paths":["","count","name"]}, - "sv_SE.person": {"paths":["","first","last","prefix","sex"],"reads":["sv_SE.first-name","sv_SE.last-name","sv_SE.title"]}, - "sv_SE.personnummer": {"paths":[""],"reads":["sv_SE.birth-number"]}, + "sv_SE.person": {"paths":["","first","last","prefix","sex"]}, + "sv_SE.personnummer": {"paths":[""]}, "sv_SE.phone": {"paths":["","a","b","c","prefix"]}, "sv_SE.price": {"paths":["","amt","ore"]}, - "sv_SE.samordningsnummer": {"paths":[""],"reads":["sv_SE.birth-number"]}, + "sv_SE.samordningsnummer": {"paths":[""]}, "sv_SE.sentence": {"paths":["","adj","noun","prep","verb"]}, "sv_SE.sex": {"paths":["","birth-number","birth-number.number","birth-number.sex","code","first-name","first-name.count","first-name.name","first-name.sex","name"]}, "sv_SE.time": {"paths":[""]}, diff --git a/datatype.go b/datatype.go index 42939cc..03f18b5 100644 --- a/datatype.go +++ b/datatype.go @@ -7,6 +7,7 @@ import ( "github.com/larvit/fejkdata/internal/datatype" "github.com/larvit/fejkdata/internal/grammar" "github.com/larvit/fejkdata/internal/invariant" + "github.com/larvit/fejkdata/internal/jsonvalue" ) // DataType is what a record column holds, which decides how a record writes its value. It @@ -29,7 +30,7 @@ func datatypeOf(m map[string]any, pos position) (*DataType, error) { } name, ok := v.(string) if !ok { - return nil, fmt.Errorf("datatype must be a string, not %s", jsonKind(v)) + return nil, fmt.Errorf("datatype must be a string, not %s", jsonvalue.Kind(v)) } for d := DataTypeString; d < datatype.Count; d++ { if name != d.String() { diff --git a/docs/decisions.md b/docs/decisions.md index 7e67314..24f9dfd 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -154,29 +154,32 @@ still lives in `data/`; `--data-path` layers over it. Valid while the shipped data stays under 100 MiB, which a gate test holds (`todo.md` item 192): `go get` fetches the whole Go module, and the `go` command refuses a Go module zip over 500 MiB (`MaxZipFile` in `golang.org/x/mod/zip`). Past it, `todo.md` item 193 moves the largest packages into nested Go modules that keep their import paths, each requiring the core release that drops the package, so no build sees it twice. -## A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads and every indexed category depending on what it provides or a later source replaced +## A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads -2026-10-03, a restructure the maintainer approved; widened to any source on 2026-10-09. +2026-10-03, a restructure the maintainer approved; widened to any source on 2026-10-09, +and narrowed by the maintainer on 2026-10-10. Goal 13: `New` pays nothing for a category a run never reaches. A manifest, -`.fejkdata.json` at a source's root, may carry an index: each category's parent table, -the paths `List` advertises below it and the categories its templates reference. A -category of an indexed source loads with every category it references and its whole -table family, and binds as a whole load does, so it renders the same as after a whole -load. `go generate` writes the shipped set's manifest from `data/`; -`TestShippedManifestIsCurrent` fails while it is stale. CI loads the whole shipped set, -and each shipped category alone into a fresh `New`, so none fails at first reach. +`.fejkdata.json` at a source's root, may carry an index: each category's parent table and +the paths `List` advertises below it. A category of an indexed source loads with every +category it references and its whole table family, and binds as a whole load does, so +it renders the same as after a whole load. `go generate` writes the shipped set's +manifest from `data/`; `TestShippedManifestIsCurrent` fails while it is stale. CI loads +the whole shipped set, and each shipped category alone into a fresh `New`, so none fails +at first reach. -Goal 6: a source without an index loads whole in `New`, so each mistake in it is `New`'s -error. So does every indexed category it reads, and every indexed category depending, -through references or table links, on a category it provides or a later source -replaced, so a replacement or a stand-in breaking a reader fails `New`. Goal 6.3: an -indexed source from anyone else may fail at first reach; `Fake` then returns the load's -error, matching `ErrLoad`, the same every call. A first reach checks only a table's -`parent` against its entry, so a table loads with its parent and a failed load puts -back whole. Proving the rest of an index is its author's, with the tool -`todo.md` item 194 plans: checked against the merged tree, an entry's paths and reads -would blame the manifest for a table or a stand-in another source adds. +Goal 6: a source without an index loads whole in `New`, with every indexed category it +reads, so each mistake in it is `New`'s error. Goal 6.3: an indexed source from anyone +else may fail at first reach; the call then returns the load's error, matching +`ErrLoad`, the same every call. So does an indexed category reading one that another +source replaced, where the replacement breaks it. The maintainer chose that on +2026-10-10 over loading in `New` every indexed category depending on a replaced one: +readers of three rounds ruled that rule harder to follow (goal 3), and it needed every +entry to list what its templates read. + +A first reach checks only a table's `parent` against its entry, so a table loads with its +parent and a failed load puts back whole. Proving the rest of an index is its author's, +with the tool `todo.md` item 194 plans. Valid while CI proves the shipped set whole and each category alone. diff --git a/fejkdata.go b/fejkdata.go index b312d6f..3da4e72 100644 --- a/fejkdata.go +++ b/fejkdata.go @@ -3,6 +3,7 @@ package fejkdata import ( crand "crypto/rand" "encoding/binary" + "errors" "fmt" "io/fs" "reflect" @@ -67,6 +68,16 @@ func WithoutShippedData() Option { return func(c *config) { c.shipped = false } } +// ErrLoad marks data that fails to load: in New, or on the first call reaching a category +// of an indexed source. +var ErrLoad = errors.New("data fails to load") + +// loadError is a load's error, unchanged in its text, matching ErrLoad. +type loadError struct{ error } + +func (e loadError) Unwrap() error { return e.error } +func (loadError) Is(target error) bool { return target == ErrLoad } + // New builds a generator from the shipped data set and the options' sources, merged // in order with the last winning a name clash. Each JSON file becomes a category // named after the file (address.json -> "address") and each subdirectory a @@ -91,7 +102,7 @@ func New(opts ...Option) (*Generator, error) { return &Generator{drawState: drawstate.New(seed), root: root}, nil } -// load builds the tree New starts from. docs/decisions.md#a-source-whose-manifest-carries-an-index-loads-each-category-on-first-reach-any-other-loads-in-new-with-what-it-reads-and-every-indexed-category-depending-on-what-it-provides-or-a-later-source-replaced +// load builds the tree New starts from. docs/decisions.md#a-source-whose-manifest-carries-an-index-loads-each-category-on-first-reach-any-other-loads-in-new-with-what-it-reads func (c config) load() (folder, error) { var sources []datafiles.Source if c.shipped { diff --git a/index.go b/index.go index 04e6de9..c3803ad 100644 --- a/index.go +++ b/index.go @@ -1,26 +1,14 @@ package fejkdata import ( - "errors" "fmt" "slices" - "sort" "strings" "github.com/larvit/fejkdata/internal/datafiles" "github.com/larvit/fejkdata/internal/grammar" ) -// ErrLoad marks data that fails to load: in New, or on the first call reaching a category -// of an indexed source. -var ErrLoad = errors.New("data fails to load") - -// loadError is a load's error, unchanged in its text, matching ErrLoad. -type loadError struct{ error } - -func (e loadError) Unwrap() error { return e.error } -func (loadError) Is(target error) bool { return target == ErrLoad } - // indexed is a category an index names and no call has reached yet, and the source holding it. type indexed struct { entry datafiles.IndexEntry @@ -113,61 +101,6 @@ func needs(root *folder, s categorySite) []categoryAt { return out } -// affectedBy is every unloaded category under root whose index entry leads to one of -// paths: through its reads, its parent or a child table, and through other unloaded -// categories. -func affectedBy(root *folder, paths []string) []categoryAt { - if len(paths) == 0 { - return nil - } - unloaded := unloadedUnder(root) - readers := map[string][]string{} - for _, p := range sortedNames(unloaded) { - c := unloaded[p] - e := c.in.unloaded[c.name].entry - for _, r := range e.Reads { - readers[r] = append(readers[r], p) - } - if e.Parent != "" { - parent := categoryPath(c.dir, e.Parent) - readers[parent] = append(readers[parent], p) - readers[p] = append(readers[p], parent) - } - } - var out []categoryAt - seen := map[string]bool{} - queue := slices.Clone(paths) - sort.Strings(queue) - for ; len(queue) > 0; queue = queue[1:] { - for _, p := range readers[queue[0]] { - if c, isUnloaded := unloaded[p]; isUnloaded && !seen[p] { - seen[p] = true - out = append(out, c) - queue = append(queue, p) - } - } - } - return out -} - -func unloadedUnder(root *folder) map[string]categoryAt { - out := map[string]categoryAt{} - var walk func(g *folder, dir []string) - walk = func(g *folder, dir []string) { - for name := range g.unloaded { - c := categoryAt{dir: dir, in: g, name: name} - out[categoryPath(dir, name)] = c - } - for name, n := range g.children { - if sub, isFolder := n.(*folder); isFolder { - walk(sub, append(dir[:len(dir):len(dir)], name)) - } - } - } - walk(root, nil) - return out -} - // loadReached loads the unloaded categories among wanted and every unloaded category they // or sites need, then runs sites and all it loaded through a whole load's pipeline. On an // error it puts back what it loaded, so the next call fails the same way. diff --git a/internal/datafiles/datafiles.go b/internal/datafiles/datafiles.go index e63c9e1..5739bc7 100644 --- a/internal/datafiles/datafiles.go +++ b/internal/datafiles/datafiles.go @@ -2,13 +2,10 @@ package datafiles import ( - "encoding/json" - "errors" "fmt" "io/fs" "os" "path" - "sort" "strings" "github.com/larvit/fejkdata/internal/grammar" @@ -59,180 +56,6 @@ func (s Source) dirPath(dir []string) string { return "." } -// ManifestFile is the file at a source's root that describes the source. -const ManifestFile = ".fejkdata.json" - -// Manifest is what a source's manifest says of it. -type Manifest struct { - // Index is every category of the source by dot path; nil where the manifest has none. - Index map[string]IndexEntry `json:"index"` -} - -// IndexEntry is what an index says of a category: its parent table, "" for none, the paths -// List advertises below it, and the categories its templates reference. -type IndexEntry struct { - Parent string `json:"parent,omitempty"` - Paths []string `json:"paths"` - Reads []string `json:"reads,omitempty"` -} - -// Manifest reads the manifest at the source's root, the zero Manifest where there is none. -func (s Source) Manifest() (Manifest, error) { - if err := s.check(); err != nil { - return Manifest{}, err - } - b, err := fs.ReadFile(s.fsys, path.Join(s.base, ManifestFile)) - if errors.Is(err, fs.ErrNotExist) { - return Manifest{}, nil - } - if err != nil { - return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err) - } - m, err := decodeManifest(b) - if err != nil { - return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err) - } - return m, nil -} - -// ManifestPath is the manifest's path as errors name it. -func (s Source) ManifestPath() string { return s.labelled(path.Join(s.base, ManifestFile)) } - -// decodeManifest decodes a manifest, refusing any key or shape it does not define, and -// reports every mistake in the order of the entries naming them. -func decodeManifest(b []byte) (Manifest, error) { - var raw any - if err := json.Unmarshal(b, &raw); err != nil { - return Manifest{}, err - } - top, isObject := raw.(map[string]any) - if !isObject { - return Manifest{}, fmt.Errorf("a manifest is a JSON object") - } - var m Manifest - var errs []error - for _, k := range sortedKeys(top) { - if k != "index" { - errs = append(errs, fmt.Errorf("unknown key %q; a manifest holds index", k)) - continue - } - index, isObject := top[k].(map[string]any) - if !isObject { - errs = append(errs, fmt.Errorf("index is a JSON object of category paths")) - continue - } - m.Index = map[string]IndexEntry{} - for _, p := range sortedKeys(index) { - e, err := decodeEntry(p, index[p]) - if err != nil { - errs = append(errs, fmt.Errorf("index entry %q: %w", p, err)) - } - m.Index[p] = e - } - } - for _, p := range sortedKeys(m.Index) { - if err := checkParent(m.Index, p); err != nil { - errs = append(errs, fmt.Errorf("index entry %q: %w", p, err)) - } - } - if len(errs) > 0 { - return Manifest{}, errors.Join(errs...) - } - return m, nil -} - -func decodeEntry(p string, v any) (IndexEntry, error) { - var e IndexEntry - errs := []error{checkCategoryPath(p)} - fields, isObject := v.(map[string]any) - if !isObject { - return e, fmt.Errorf("an entry is a JSON object of parent, paths and reads") - } - if _, has := fields["paths"]; !has { - errs = append(errs, fmt.Errorf("paths is missing")) - } - var err error - for _, k := range sortedKeys(fields) { - switch k { - case "parent": - var isString bool - if e.Parent, isString = fields[k].(string); !isString { - err = fmt.Errorf("parent is a string") - } else if err = grammar.CheckIdentifier(e.Parent); err != nil { - err = fmt.Errorf("parent %w", err) - } - case "paths": - e.Paths, err = stringList(k, fields[k], func(q string) error { - if q == "" { - return nil - } - return checkCategoryPath(q) - }) - case "reads": - e.Reads, err = stringList(k, fields[k], checkCategoryPath) - default: - err = fmt.Errorf("unknown key %q; an entry holds parent, paths and reads", k) - } - errs = append(errs, err) - } - return e, errors.Join(errs...) -} - -// stringList is the JSON array of strings v, each passing check. -func stringList(key string, v any, check func(string) error) ([]string, error) { - items, isArray := v.([]any) - if !isArray { - return nil, fmt.Errorf("%s is an array of strings", key) - } - out := make([]string, 0, len(items)) - for _, item := range items { - s, isString := item.(string) - if !isString { - return nil, fmt.Errorf("%s is an array of strings", key) - } - if err := check(s); err != nil { - return nil, fmt.Errorf("%s: %w", key, err) - } - out = append(out, s) - } - return out, nil -} - -// checkParent refuses a parent naming no entry beside p: a table loads with its parent, -// from one source. -func checkParent(index map[string]IndexEntry, p string) error { - parent := index[p].Parent - if parent == "" { - return nil - } - sibling := parent - if dot := strings.LastIndexByte(p, '.'); dot >= 0 { - sibling = p[:dot+1] + parent - } - if _, ok := index[sibling]; !ok { - return fmt.Errorf("parent %q names no entry beside it in this index", parent) - } - return nil -} - -func checkCategoryPath(p string) error { - for _, seg := range strings.Split(p, ".") { - if err := grammar.CheckIdentifier(seg); err != nil { - return fmt.Errorf("%q: %w", p, err) - } - } - return nil -} - -func sortedKeys[V any](m map[string]V) []string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - sort.Strings(keys) - return keys -} - // Walk hands compile every category of the tree, folders and files in name order. A // folder holding no category anywhere below it is skipped; a hidden file or folder is no // category, though a category may name a hidden rows file beside it. diff --git a/internal/datafiles/manifest.go b/internal/datafiles/manifest.go new file mode 100644 index 0000000..25f1a2a --- /dev/null +++ b/internal/datafiles/manifest.go @@ -0,0 +1,200 @@ +package datafiles + +import ( + "errors" + "fmt" + "io/fs" + "path" + "sort" + "strings" + + "github.com/larvit/fejkdata/internal/grammar" + "github.com/larvit/fejkdata/internal/jsonvalue" +) + +// ManifestFile is the file at a source's root that describes the source. +const ManifestFile = ".fejkdata.json" + +// Manifest is what a source's manifest says of it. +type Manifest struct { + // Index is every category of the source by dot path; nil where the manifest has none. + Index map[string]IndexEntry `json:"index"` +} + +// IndexEntry is what an index says of a category: its parent table, "" for none, and the +// paths List advertises below it. +type IndexEntry struct { + Parent string `json:"parent,omitempty"` + Paths []string `json:"paths"` +} + +// Manifest reads the manifest at the source's root, the zero Manifest where there is none. +func (s Source) Manifest() (Manifest, error) { + if err := s.check(); err != nil { + return Manifest{}, err + } + b, err := fs.ReadFile(s.fsys, path.Join(s.base, ManifestFile)) + if errors.Is(err, fs.ErrNotExist) { + return Manifest{}, nil + } + if err != nil { + return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err) + } + m, err := decodeManifest(b) + if err != nil { + return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err) + } + return m, nil +} + +// ManifestPath is the manifest's path as errors name it. +func (s Source) ManifestPath() string { return s.labelled(path.Join(s.base, ManifestFile)) } + +// decodeManifest decodes a manifest, refusing any key or shape it does not define, and +// reports every mistake: first each entry's own, in path order, then each parent naming +// no entry. +func decodeManifest(b []byte) (Manifest, error) { + raw, err := jsonvalue.Decode(b) + if err != nil { + return Manifest{}, err + } + top, isObject := raw.Any().(map[string]any) + if !isObject { + return Manifest{}, fmt.Errorf("a manifest must be an object, not %s", jsonvalue.Kind(raw.Any())) + } + var m Manifest + var errs []error + for _, k := range sortedKeys(top) { + if k != "index" { + errs = append(errs, fmt.Errorf("unknown key %q; a manifest holds index", k)) + continue + } + var indexErrs []error + m.Index, indexErrs = decodeIndex(top[k]) + errs = append(errs, indexErrs...) + } + return m, errors.Join(errs...) +} + +func decodeIndex(v any) (map[string]IndexEntry, []error) { + entries, isObject := v.(map[string]any) + if !isObject { + return nil, []error{fmt.Errorf("index must be an object, not %s", jsonvalue.Kind(v))} + } + index := map[string]IndexEntry{} + var errs []error + for _, p := range sortedKeys(entries) { + e, err := decodeEntry(p, entries[p]) + if err != nil { + errs = append(errs, fmt.Errorf("index entry %q: %w", p, err)) + } + index[p] = e + } + for _, p := range sortedKeys(index) { + if err := checkParent(index, p); err != nil { + errs = append(errs, fmt.Errorf("index entry %q: %w", p, err)) + } + } + return index, errs +} + +// decodeEntry decodes one entry, leaving out a parent it refuses, so checkParent names no +// mistake twice. +func decodeEntry(p string, v any) (IndexEntry, error) { + var e IndexEntry + errs := []error{checkCategoryPath(p)} + fields, isObject := v.(map[string]any) + if !isObject { + errs = append(errs, fmt.Errorf("an entry must be an object, not %s", jsonvalue.Kind(v))) + return e, errors.Join(errs...) + } + if _, has := fields["paths"]; !has { + errs = append(errs, fmt.Errorf("paths is missing")) + } + for _, k := range sortedKeys(fields) { + switch k { + case "parent": + parent, err := decodeParent(fields[k]) + e.Parent = parent + errs = append(errs, err) + case "paths": + var err error + e.Paths, err = decodePaths(fields[k]) + errs = append(errs, err) + default: + errs = append(errs, fmt.Errorf("unknown key %q; an entry holds parent and paths", k)) + } + } + return e, errors.Join(errs...) +} + +func decodeParent(v any) (string, error) { + parent, isString := v.(string) + if !isString { + return "", fmt.Errorf("parent must be a string, not %s", jsonvalue.Kind(v)) + } + if err := grammar.CheckIdentifier(parent); err != nil { + return "", fmt.Errorf("parent %w", err) + } + return parent, nil +} + +// decodePaths decodes a list of paths below a category, "" being the category itself. +func decodePaths(v any) ([]string, error) { + items, isList := v.([]any) + if !isList { + return nil, fmt.Errorf("paths must be a list, not %s", jsonvalue.Kind(v)) + } + out := make([]string, 0, len(items)) + var errs []error + for i, item := range items { + q, isString := item.(string) + if !isString { + errs = append(errs, fmt.Errorf("paths item %d must be a string, not %s", i+1, jsonvalue.Kind(item))) + continue + } + if q != "" { + if err := checkCategoryPath(q); err != nil { + errs = append(errs, fmt.Errorf("paths item %d: %w", i+1, err)) + continue + } + } + out = append(out, q) + } + return out, errors.Join(errs...) +} + +// checkParent refuses a parent naming no entry beside p: a table loads with its parent, +// from one source. +func checkParent(index map[string]IndexEntry, p string) error { + parent := index[p].Parent + if parent == "" { + return nil + } + sibling := parent + if dot := strings.LastIndexByte(p, '.'); dot >= 0 { + sibling = p[:dot+1] + parent + } + if _, ok := index[sibling]; !ok { + return fmt.Errorf("parent %q names no entry beside it in this index", parent) + } + return nil +} + +func checkCategoryPath(p string) error { + for _, seg := range strings.Split(p, ".") { + if err := grammar.CheckIdentifier(seg); err != nil { + return fmt.Errorf("%q: %w", p, err) + } + } + return nil +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/internal/jsonvalue/jsonvalue.go b/internal/jsonvalue/jsonvalue.go index c477f51..17ba339 100644 --- a/internal/jsonvalue/jsonvalue.go +++ b/internal/jsonvalue/jsonvalue.go @@ -4,6 +4,7 @@ package jsonvalue import ( "bytes" "encoding/json" + "fmt" ) // Value is a JSON value that only Decode, String and Object make, so a template never compiles @@ -37,3 +38,22 @@ func Object(fields map[string]Value) Value { // Any is the value as encoding/json decodes it: a map[string]any, []any, string, json.Number, // bool or nil. func (v Value) Any() any { return v.v } + +// Kind names the kind of a value as Any gives it, such as "a number" or "a list". +func Kind(v any) string { + switch v.(type) { + case json.Number: + return "a number" + case bool: + return "a boolean" + case string: + return "a string" + case []any: + return "a list" + case map[string]any: + return "an object" + case nil: + return "null" + } + return fmt.Sprintf("%T", v) +} diff --git a/node.go b/node.go index b25035b..dc9eeda 100644 --- a/node.go +++ b/node.go @@ -193,22 +193,6 @@ func typedOrText(text string, ds ...DataType) error { // jsonKind names the kind of a JSON value an option cannot take, in the data format's own // terms rather than the decoding library's. -func jsonKind(v any) string { - switch v.(type) { - case json.Number: - return "a number" - case bool: - return "a boolean" - case string: - return "a string" - case []any: - return "a list" - case map[string]any: - return "an object" - } - return fmt.Sprintf("%T", v) -} - func compileString(s string) (*template, error) { toks, unbound, err := parseChecked(s, nil) if err != nil { @@ -332,7 +316,7 @@ func readOptions(m map[string]any, pos position) (templateOptions, error) { } if sv, ok := m["separator"]; ok { if o.separator, ok = sv.(string); !ok { - return o, fmt.Errorf("separator must be a string, not %s", jsonKind(sv)) + return o, fmt.Errorf("separator must be a string, not %s", jsonvalue.Kind(sv)) } } return o, nil @@ -365,7 +349,7 @@ func repeatOf(m map[string]any) (int, error) { } r, ok := numberOf(rv) if !ok { - return 0, fmt.Errorf("repeat must be a number, not %s", jsonKind(rv)) + return 0, fmt.Errorf("repeat must be a number, not %s", jsonvalue.Kind(rv)) } if err := notTiny("repeat", rv, r); err != nil { return 0, err @@ -392,7 +376,7 @@ func weightOf(raw any) (float64, error) { } w, ok := numberOf(wv) if !ok { - return 0, fmt.Errorf(`weight is an option and takes a number, not %s; a field cannot be named "weight"`, jsonKind(wv)) + return 0, fmt.Errorf(`weight is an option and takes a number, not %s; a field cannot be named "weight"`, jsonvalue.Kind(wv)) } if w < 0 || math.IsInf(w, 0) { return 0, fmt.Errorf("weight must be a finite number of 0 or more, got %v", wv) diff --git a/perf_test.go b/perf_test.go index 996caae..dfd95b0 100644 --- a/perf_test.go +++ b/perf_test.go @@ -177,7 +177,7 @@ func BenchmarkWideTokens500(b *testing.B) { // New decodes the shipped manifest and loads nothing else. func TestNoNewAllocRegression(t *testing.T) { - const base = 2501.0 + const base = 2449.0 allocs := testing.AllocsPerRun(100, func() { if _, err := New(WithSeed(1)); err != nil { t.Fatal(err) diff --git a/shipped_test.go b/shipped_test.go index bba4ad0..1b21c1b 100644 --- a/shipped_test.go +++ b/shipped_test.go @@ -263,7 +263,7 @@ func TestABrokenManifestFailsNewNamingEveryMistake(t *testing.T) { `{`: {datafiles.ManifestFile}, `{"index": {}} {}`: {"after top-level value"}, `{"indx": {}, "INDEX": {}}`: {`unknown key "INDEX"`, `unknown key "indx"`}, - `{"index": []}`: {"index must be an object, not an array"}, + `{"index": []}`: {"index must be an object, not a list"}, `{"index": {"a..b": 5}}`: {`"a..b"`, "must be an object, not a number"}, `{"index": {"a": {"Paths": [""]}}}`: {`unknown key "Paths"`, "paths is missing"}, `{"index": {"a": {}, "b": {}}}`: {`"a": paths is missing`, `"b": paths is missing`}, diff --git a/table.go b/table.go index d02ddb8..5221135 100644 --- a/table.go +++ b/table.go @@ -7,6 +7,7 @@ import ( "github.com/larvit/fejkdata/internal/drawstate" "github.com/larvit/fejkdata/internal/grammar" "github.com/larvit/fejkdata/internal/invariant" + "github.com/larvit/fejkdata/internal/jsonvalue" "github.com/larvit/fejkdata/internal/rows" ) @@ -102,7 +103,7 @@ func readTableOptions(m map[string]any) (tableOptionValues, error) { } s, isString := v.(string) if !isString { - return o, fmt.Errorf("%s must be a string, not %s", k, jsonKind(v)) + return o, fmt.Errorf("%s must be a string, not %s", k, jsonvalue.Kind(v)) } if k != "format" && s == "" { return o, fmt.Errorf("%s names a column, so it cannot be empty; drop it", k)