Make great documentation goal 1.2, and plan the README's reorganisation and the nested-template gaps #33
@@ -47,7 +47,7 @@ In [docs/decisions.md](docs/decisions.md):
|
||||
- `FakeTemplate` and `NewTemplate` both stay
|
||||
- The shipped data is embedded, not discovered
|
||||
- The shipped data is Go packages a library imports by choice, and the CLI carries every one, from 0.1.0
|
||||
- A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads and every indexed category depending on what it provides or a later source replaced
|
||||
- A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads
|
||||
- Every `{…}` draws afresh, and only a name keeps a pick
|
||||
- Reference sigils follow the filesystem
|
||||
- A change to what exists is a major; a minor only adds
|
||||
|
||||
+2
-2
@@ -128,5 +128,5 @@ replacement, and each removed path, column or flag.
|
||||
- A data source whose root holds a `.fejkdata.json` manifest with an `index` loads each
|
||||
category on the first call reaching it, as the shipped set does. Data failing to load,
|
||||
in `New` or there, gives an error matching `ErrLoad`, and the CLI exits 1. A source without
|
||||
one loads in `New`, with the shipped categories it reads and every shipped category
|
||||
depending on one it replaces; the rest of the shipped set still loads on first reach.
|
||||
one loads in `New`, with the shipped categories it reads; the rest of the shipped set,
|
||||
a reader of a category it replaces too, loads on first reach.
|
||||
|
||||
@@ -174,13 +174,13 @@ a category may name a hidden rows file beside it.
|
||||
|
||||
A source whose root holds `.fejkdata.json` with an `index` loads each category on the
|
||||
first call reaching it, and its author proves each loads. Any other source loads in
|
||||
`New`, and so does every indexed category that depends on what such a source provides
|
||||
or on a category a later source replaced. The index names every category of its source,
|
||||
so a category file it leaves out never loads. The shipped set's
|
||||
[`data/.fejkdata.json`](data/.fejkdata.json) shows the shape: each category's dot path,
|
||||
its `parent` table, an entry of the same index, every path `List` advertises below it,
|
||||
`""` for the category itself, and the other categories its templates `reads`. A table
|
||||
whose parent column differs from its entry's `parent` fails at first reach.
|
||||
`New`, with the indexed categories it reads. A shipped category reading one that another
|
||||
source replaced loads on its own first reach, and fails there if the replacement breaks
|
||||
it. The index names every category of its source, so a category file it leaves out never
|
||||
loads. The shipped set's [`data/.fejkdata.json`](data/.fejkdata.json) shows the shape:
|
||||
each category's dot path, its `parent` table, an entry of the same index, and every path
|
||||
`List` advertises below it, `""` for the category itself. A table whose parent column
|
||||
differs from its entry's `parent` fails at first reach.
|
||||
`//go:embed` of a directory leaves the manifest out; name it in the pattern too.
|
||||
|
||||
Each locale carries `address`, `color`, `company`, `date`, `email`, `first-name`,
|
||||
|
||||
@@ -7,67 +7,33 @@ import (
|
||||
)
|
||||
|
||||
// loadSources merges sources in order, the last winning a clash. A source whose manifest
|
||||
// carries an index leaves its categories unloaded; every other source loads here. So do
|
||||
// the unloaded categories that depend on one it loaded or on one a later source replaced,
|
||||
// so that a reader such a change breaks fails New, not a first reach.
|
||||
// carries an index leaves its categories unloaded; every other source loads here, with
|
||||
// the unloaded categories it needs.
|
||||
func loadSources(sources []datafiles.Source) (folder, error) {
|
||||
root := folder{children: map[string]node{}}
|
||||
var changed []string
|
||||
for _, src := range sources {
|
||||
replaced, err := addSource(&root, src)
|
||||
if err != nil {
|
||||
if err := addSource(&root, src); err != nil {
|
||||
return folder{}, err
|
||||
}
|
||||
changed = append(changed, replaced...)
|
||||
}
|
||||
sites := categorySites(&root)
|
||||
for _, s := range sites {
|
||||
changed = append(changed, s.path)
|
||||
}
|
||||
return root, loadReached(&root, sites, affectedBy(&root, changed))
|
||||
return root, loadReached(&root, categorySites(&root), nil)
|
||||
}
|
||||
|
||||
// addSource adds the categories of src to root, and returns the path of each category
|
||||
// already there that src replaced or put out of reach.
|
||||
func addSource(root *folder, src datafiles.Source) ([]string, error) {
|
||||
func addSource(root *folder, src datafiles.Source) error {
|
||||
m, err := src.Manifest()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return err
|
||||
}
|
||||
before := standing(root)
|
||||
if m.Index != nil {
|
||||
placeIndex(root, src, m.Index)
|
||||
} else {
|
||||
g := &folder{children: map[string]node{}}
|
||||
if err := src.Walk(compileInto(func(dir []string) *folder { return madeFolder(g, dir) })); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mergeFolder(root, g)
|
||||
return nil
|
||||
}
|
||||
if len(before) == 0 {
|
||||
return nil, nil
|
||||
g := &folder{children: map[string]node{}}
|
||||
if err := src.Walk(compileInto(func(dir []string) *folder { return madeFolder(g, dir) })); err != nil {
|
||||
return err
|
||||
}
|
||||
after := standing(root)
|
||||
var replaced []string
|
||||
for p, was := range before {
|
||||
if after[p] != was {
|
||||
replaced = append(replaced, p)
|
||||
}
|
||||
}
|
||||
return replaced, nil
|
||||
}
|
||||
|
||||
// standing is what stands at the path of every category under root: its node, or its
|
||||
// unloaded entry.
|
||||
func standing(root *folder) map[string]any {
|
||||
out := map[string]any{}
|
||||
for _, s := range categorySites(root) {
|
||||
out[s.path] = s.n
|
||||
}
|
||||
for p, c := range unloadedUnder(root) {
|
||||
out[p] = c.in.unloaded[c.name]
|
||||
}
|
||||
return out
|
||||
mergeFolder(root, g)
|
||||
return nil
|
||||
}
|
||||
|
||||
func compileInto(place func(dir []string) *folder) func(datafiles.Category) error {
|
||||
|
||||
+9
-9
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"index": {
|
||||
"en_US.address": {"paths":["","locality","postal-code","region","street","street-number"],"reads":["geo.US.address"]},
|
||||
"en_US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
|
||||
"en_US.color": {"paths":[""]},
|
||||
"en_US.company": {"paths":["","base","suffix"]},
|
||||
"en_US.date": {"paths":[""]},
|
||||
@@ -9,8 +9,8 @@
|
||||
"en_US.ip": {"paths":[""]},
|
||||
"en_US.itin": {"paths":["","area","group","serial"]},
|
||||
"en_US.last-name": {"paths":["","count","name"]},
|
||||
"en_US.person": {"paths":["","first","last","prefix","sex"],"reads":["en_US.first-name","en_US.last-name"]},
|
||||
"en_US.phone": {"paths":["","area","exchange","line"],"reads":["en_US.phone-exchange"]},
|
||||
"en_US.person": {"paths":["","first","last","prefix","sex"]},
|
||||
"en_US.phone": {"paths":["","area","exchange","line"]},
|
||||
"en_US.phone-exchange": {"paths":[""]},
|
||||
"en_US.price": {"paths":["","amt","cents"]},
|
||||
"en_US.sentence": {"paths":["","adj","noun","prep","verb"]},
|
||||
@@ -22,13 +22,13 @@
|
||||
"en_US.username": {"paths":["","n"]},
|
||||
"en_US.version": {"paths":["","n","pre","suffix"]},
|
||||
"en_US.word": {"paths":[""]},
|
||||
"geo.SE.address": {"paths":["","locality","postal-code","street","street-number"],"reads":["geo.SE.locality"]},
|
||||
"geo.SE.address": {"paths":["","locality","postal-code","street","street-number"]},
|
||||
"geo.SE.locality": {"parent":"municipality","paths":["","lat","lon","municipality","name","population","postal-code","postal-code.code","postal-code.locality","street","street.locality","street.name","street.segments"]},
|
||||
"geo.SE.municipality": {"parent":"region","paths":["","code","locality","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.locality","locality.street.name","locality.street.segments","name","population","region"]},
|
||||
"geo.SE.postal-code": {"parent":"locality","paths":["","code","locality"]},
|
||||
"geo.SE.region": {"paths":["","code","municipality","municipality.code","municipality.locality","municipality.locality.lat","municipality.locality.lon","municipality.locality.municipality","municipality.locality.name","municipality.locality.population","municipality.locality.postal-code","municipality.locality.postal-code.code","municipality.locality.postal-code.locality","municipality.locality.street","municipality.locality.street.locality","municipality.locality.street.name","municipality.locality.street.segments","municipality.name","municipality.population","municipality.region","name","population","timezone"]},
|
||||
"geo.SE.street": {"parent":"locality","paths":["","locality","name","segments"]},
|
||||
"geo.US.address": {"paths":["","locality","postal-code","region","street","street-number"],"reads":["geo.US.locality"]},
|
||||
"geo.US.address": {"paths":["","locality","postal-code","region","street","street-number"]},
|
||||
"geo.US.locality": {"parent":"municipality","paths":["","code","lat","lon","municipality","name","population","postal-code","postal-code.addresses","postal-code.code","postal-code.locality","street","street.addresses","street.locality","street.name"]},
|
||||
"geo.US.municipality": {"parent":"region","paths":["","code","locality","locality.code","locality.lat","locality.lon","locality.municipality","locality.name","locality.population","locality.postal-code","locality.postal-code.addresses","locality.postal-code.code","locality.postal-code.locality","locality.street","locality.street.addresses","locality.street.locality","locality.street.name","name","population","region"]},
|
||||
"geo.US.postal-code": {"parent":"locality","paths":["","addresses","code","locality"]},
|
||||
@@ -54,7 +54,7 @@
|
||||
"misc.tld": {"paths":["","tld","type","unicode"]},
|
||||
"misc.useragent": {"paths":["","browser","device","os","ua"]},
|
||||
"misc.uuid": {"paths":["","variant"]},
|
||||
"sv_SE.address": {"paths":["","locality","postal-code","street","street-number"],"reads":["geo.SE.address"]},
|
||||
"sv_SE.address": {"paths":["","locality","postal-code","street","street-number"]},
|
||||
"sv_SE.birth-number": {"parent":"sex","paths":["","number","sex"]},
|
||||
"sv_SE.color": {"paths":[""]},
|
||||
"sv_SE.company": {"paths":["","base","suffix"]},
|
||||
@@ -63,11 +63,11 @@
|
||||
"sv_SE.first-name": {"parent":"sex","paths":["","count","name","sex"]},
|
||||
"sv_SE.ip": {"paths":[""]},
|
||||
"sv_SE.last-name": {"paths":["","count","name"]},
|
||||
"sv_SE.person": {"paths":["","first","last","prefix","sex"],"reads":["sv_SE.first-name","sv_SE.last-name","sv_SE.title"]},
|
||||
"sv_SE.personnummer": {"paths":[""],"reads":["sv_SE.birth-number"]},
|
||||
"sv_SE.person": {"paths":["","first","last","prefix","sex"]},
|
||||
"sv_SE.personnummer": {"paths":[""]},
|
||||
"sv_SE.phone": {"paths":["","a","b","c","prefix"]},
|
||||
"sv_SE.price": {"paths":["","amt","ore"]},
|
||||
"sv_SE.samordningsnummer": {"paths":[""],"reads":["sv_SE.birth-number"]},
|
||||
"sv_SE.samordningsnummer": {"paths":[""]},
|
||||
"sv_SE.sentence": {"paths":["","adj","noun","prep","verb"]},
|
||||
"sv_SE.sex": {"paths":["","birth-number","birth-number.number","birth-number.sex","code","first-name","first-name.count","first-name.name","first-name.sex","name"]},
|
||||
"sv_SE.time": {"paths":[""]},
|
||||
|
||||
+2
-1
@@ -7,6 +7,7 @@ import (
|
||||
"github.com/larvit/fejkdata/internal/datatype"
|
||||
"github.com/larvit/fejkdata/internal/grammar"
|
||||
"github.com/larvit/fejkdata/internal/invariant"
|
||||
"github.com/larvit/fejkdata/internal/jsonvalue"
|
||||
)
|
||||
|
||||
// DataType is what a record column holds, which decides how a record writes its value. It
|
||||
@@ -29,7 +30,7 @@ func datatypeOf(m map[string]any, pos position) (*DataType, error) {
|
||||
}
|
||||
name, ok := v.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("datatype must be a string, not %s", jsonKind(v))
|
||||
return nil, fmt.Errorf("datatype must be a string, not %s", jsonvalue.Kind(v))
|
||||
}
|
||||
for d := DataTypeString; d < datatype.Count; d++ {
|
||||
if name != d.String() {
|
||||
|
||||
+22
-19
@@ -154,29 +154,32 @@ still lives in `data/`; `--data-path` layers over it.
|
||||
|
||||
Valid while the shipped data stays under 100 MiB, which a gate test holds (`todo.md` item 192): `go get` fetches the whole Go module, and the `go` command refuses a Go module zip over 500 MiB (`MaxZipFile` in `golang.org/x/mod/zip`). Past it, `todo.md` item 193 moves the largest packages into nested Go modules that keep their import paths, each requiring the core release that drops the package, so no build sees it twice.
|
||||
|
||||
## A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads and every indexed category depending on what it provides or a later source replaced
|
||||
## A source whose manifest carries an index loads each category on first reach; any other loads in `New`, with what it reads
|
||||
|
||||
2026-10-03, a restructure the maintainer approved; widened to any source on 2026-10-09.
|
||||
2026-10-03, a restructure the maintainer approved; widened to any source on 2026-10-09,
|
||||
and narrowed by the maintainer on 2026-10-10.
|
||||
|
||||
Goal 13: `New` pays nothing for a category a run never reaches. A manifest,
|
||||
`.fejkdata.json` at a source's root, may carry an index: each category's parent table,
|
||||
the paths `List` advertises below it and the categories its templates reference. A
|
||||
category of an indexed source loads with every category it references and its whole
|
||||
table family, and binds as a whole load does, so it renders the same as after a whole
|
||||
load. `go generate` writes the shipped set's manifest from `data/`;
|
||||
`TestShippedManifestIsCurrent` fails while it is stale. CI loads the whole shipped set,
|
||||
and each shipped category alone into a fresh `New`, so none fails at first reach.
|
||||
`.fejkdata.json` at a source's root, may carry an index: each category's parent table and
|
||||
the paths `List` advertises below it. A category of an indexed source loads with every
|
||||
category it references and its whole table family, and binds as a whole load does, so
|
||||
it renders the same as after a whole load. `go generate` writes the shipped set's
|
||||
manifest from `data/`; `TestShippedManifestIsCurrent` fails while it is stale. CI loads
|
||||
the whole shipped set, and each shipped category alone into a fresh `New`, so none fails
|
||||
at first reach.
|
||||
|
||||
Goal 6: a source without an index loads whole in `New`, so each mistake in it is `New`'s
|
||||
error. So does every indexed category it reads, and every indexed category depending,
|
||||
through references or table links, on a category it provides or a later source
|
||||
replaced, so a replacement or a stand-in breaking a reader fails `New`. Goal 6.3: an
|
||||
indexed source from anyone else may fail at first reach; `Fake` then returns the load's
|
||||
error, matching `ErrLoad`, the same every call. A first reach checks only a table's
|
||||
`parent` against its entry, so a table loads with its parent and a failed load puts
|
||||
back whole. Proving the rest of an index is its author's, with the tool
|
||||
`todo.md` item 194 plans: checked against the merged tree, an entry's paths and reads
|
||||
would blame the manifest for a table or a stand-in another source adds.
|
||||
Goal 6: a source without an index loads whole in `New`, with every indexed category it
|
||||
reads, so each mistake in it is `New`'s error. Goal 6.3: an indexed source from anyone
|
||||
else may fail at first reach; the call then returns the load's error, matching
|
||||
`ErrLoad`, the same every call. So does an indexed category reading one that another
|
||||
source replaced, where the replacement breaks it. The maintainer chose that on
|
||||
2026-10-10 over loading in `New` every indexed category depending on a replaced one:
|
||||
readers of three rounds ruled that rule harder to follow (goal 3), and it needed every
|
||||
entry to list what its templates read.
|
||||
|
||||
A first reach checks only a table's `parent` against its entry, so a table loads with its
|
||||
parent and a failed load puts back whole. Proving the rest of an index is its author's,
|
||||
with the tool `todo.md` item 194 plans.
|
||||
|
||||
Valid while CI proves the shipped set whole and each category alone.
|
||||
|
||||
|
||||
+12
-1
@@ -3,6 +3,7 @@ package fejkdata
|
||||
import (
|
||||
crand "crypto/rand"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"reflect"
|
||||
@@ -67,6 +68,16 @@ func WithoutShippedData() Option {
|
||||
return func(c *config) { c.shipped = false }
|
||||
}
|
||||
|
||||
// ErrLoad marks data that fails to load: in New, or on the first call reaching a category
|
||||
// of an indexed source.
|
||||
var ErrLoad = errors.New("data fails to load")
|
||||
|
||||
// loadError is a load's error, unchanged in its text, matching ErrLoad.
|
||||
type loadError struct{ error }
|
||||
|
||||
func (e loadError) Unwrap() error { return e.error }
|
||||
func (loadError) Is(target error) bool { return target == ErrLoad }
|
||||
|
||||
// New builds a generator from the shipped data set and the options' sources, merged
|
||||
// in order with the last winning a name clash. Each JSON file becomes a category
|
||||
// named after the file (address.json -> "address") and each subdirectory a
|
||||
@@ -91,7 +102,7 @@ func New(opts ...Option) (*Generator, error) {
|
||||
return &Generator{drawState: drawstate.New(seed), root: root}, nil
|
||||
}
|
||||
|
||||
// load builds the tree New starts from. docs/decisions.md#a-source-whose-manifest-carries-an-index-loads-each-category-on-first-reach-any-other-loads-in-new-with-what-it-reads-and-every-indexed-category-depending-on-what-it-provides-or-a-later-source-replaced
|
||||
// load builds the tree New starts from. docs/decisions.md#a-source-whose-manifest-carries-an-index-loads-each-category-on-first-reach-any-other-loads-in-new-with-what-it-reads
|
||||
func (c config) load() (folder, error) {
|
||||
var sources []datafiles.Source
|
||||
if c.shipped {
|
||||
|
||||
@@ -1,26 +1,14 @@
|
||||
package fejkdata
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/larvit/fejkdata/internal/datafiles"
|
||||
"github.com/larvit/fejkdata/internal/grammar"
|
||||
)
|
||||
|
||||
// ErrLoad marks data that fails to load: in New, or on the first call reaching a category
|
||||
// of an indexed source.
|
||||
var ErrLoad = errors.New("data fails to load")
|
||||
|
||||
// loadError is a load's error, unchanged in its text, matching ErrLoad.
|
||||
type loadError struct{ error }
|
||||
|
||||
func (e loadError) Unwrap() error { return e.error }
|
||||
func (loadError) Is(target error) bool { return target == ErrLoad }
|
||||
|
||||
// indexed is a category an index names and no call has reached yet, and the source holding it.
|
||||
type indexed struct {
|
||||
entry datafiles.IndexEntry
|
||||
@@ -113,61 +101,6 @@ func needs(root *folder, s categorySite) []categoryAt {
|
||||
return out
|
||||
}
|
||||
|
||||
// affectedBy is every unloaded category under root whose index entry leads to one of
|
||||
// paths: through its reads, its parent or a child table, and through other unloaded
|
||||
// categories.
|
||||
func affectedBy(root *folder, paths []string) []categoryAt {
|
||||
if len(paths) == 0 {
|
||||
return nil
|
||||
}
|
||||
unloaded := unloadedUnder(root)
|
||||
readers := map[string][]string{}
|
||||
for _, p := range sortedNames(unloaded) {
|
||||
c := unloaded[p]
|
||||
e := c.in.unloaded[c.name].entry
|
||||
for _, r := range e.Reads {
|
||||
readers[r] = append(readers[r], p)
|
||||
}
|
||||
if e.Parent != "" {
|
||||
parent := categoryPath(c.dir, e.Parent)
|
||||
readers[parent] = append(readers[parent], p)
|
||||
readers[p] = append(readers[p], parent)
|
||||
}
|
||||
}
|
||||
var out []categoryAt
|
||||
seen := map[string]bool{}
|
||||
queue := slices.Clone(paths)
|
||||
sort.Strings(queue)
|
||||
for ; len(queue) > 0; queue = queue[1:] {
|
||||
for _, p := range readers[queue[0]] {
|
||||
if c, isUnloaded := unloaded[p]; isUnloaded && !seen[p] {
|
||||
seen[p] = true
|
||||
out = append(out, c)
|
||||
queue = append(queue, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func unloadedUnder(root *folder) map[string]categoryAt {
|
||||
out := map[string]categoryAt{}
|
||||
var walk func(g *folder, dir []string)
|
||||
walk = func(g *folder, dir []string) {
|
||||
for name := range g.unloaded {
|
||||
c := categoryAt{dir: dir, in: g, name: name}
|
||||
out[categoryPath(dir, name)] = c
|
||||
}
|
||||
for name, n := range g.children {
|
||||
if sub, isFolder := n.(*folder); isFolder {
|
||||
walk(sub, append(dir[:len(dir):len(dir)], name))
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(root, nil)
|
||||
return out
|
||||
}
|
||||
|
||||
// loadReached loads the unloaded categories among wanted and every unloaded category they
|
||||
// or sites need, then runs sites and all it loaded through a whole load's pipeline. On an
|
||||
// error it puts back what it loaded, so the next call fails the same way.
|
||||
|
||||
@@ -2,13 +2,10 @@
|
||||
package datafiles
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/larvit/fejkdata/internal/grammar"
|
||||
@@ -59,180 +56,6 @@ func (s Source) dirPath(dir []string) string {
|
||||
return "."
|
||||
}
|
||||
|
||||
// ManifestFile is the file at a source's root that describes the source.
|
||||
const ManifestFile = ".fejkdata.json"
|
||||
|
||||
// Manifest is what a source's manifest says of it.
|
||||
type Manifest struct {
|
||||
// Index is every category of the source by dot path; nil where the manifest has none.
|
||||
Index map[string]IndexEntry `json:"index"`
|
||||
}
|
||||
|
||||
// IndexEntry is what an index says of a category: its parent table, "" for none, the paths
|
||||
// List advertises below it, and the categories its templates reference.
|
||||
type IndexEntry struct {
|
||||
Parent string `json:"parent,omitempty"`
|
||||
Paths []string `json:"paths"`
|
||||
Reads []string `json:"reads,omitempty"`
|
||||
}
|
||||
|
||||
// Manifest reads the manifest at the source's root, the zero Manifest where there is none.
|
||||
func (s Source) Manifest() (Manifest, error) {
|
||||
if err := s.check(); err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
b, err := fs.ReadFile(s.fsys, path.Join(s.base, ManifestFile))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return Manifest{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err)
|
||||
}
|
||||
m, err := decodeManifest(b)
|
||||
if err != nil {
|
||||
return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// ManifestPath is the manifest's path as errors name it.
|
||||
func (s Source) ManifestPath() string { return s.labelled(path.Join(s.base, ManifestFile)) }
|
||||
|
||||
// decodeManifest decodes a manifest, refusing any key or shape it does not define, and
|
||||
// reports every mistake in the order of the entries naming them.
|
||||
func decodeManifest(b []byte) (Manifest, error) {
|
||||
var raw any
|
||||
if err := json.Unmarshal(b, &raw); err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
top, isObject := raw.(map[string]any)
|
||||
if !isObject {
|
||||
return Manifest{}, fmt.Errorf("a manifest is a JSON object")
|
||||
}
|
||||
var m Manifest
|
||||
var errs []error
|
||||
for _, k := range sortedKeys(top) {
|
||||
if k != "index" {
|
||||
errs = append(errs, fmt.Errorf("unknown key %q; a manifest holds index", k))
|
||||
continue
|
||||
}
|
||||
index, isObject := top[k].(map[string]any)
|
||||
if !isObject {
|
||||
errs = append(errs, fmt.Errorf("index is a JSON object of category paths"))
|
||||
continue
|
||||
}
|
||||
m.Index = map[string]IndexEntry{}
|
||||
for _, p := range sortedKeys(index) {
|
||||
e, err := decodeEntry(p, index[p])
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("index entry %q: %w", p, err))
|
||||
}
|
||||
m.Index[p] = e
|
||||
}
|
||||
}
|
||||
for _, p := range sortedKeys(m.Index) {
|
||||
if err := checkParent(m.Index, p); err != nil {
|
||||
errs = append(errs, fmt.Errorf("index entry %q: %w", p, err))
|
||||
}
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
return Manifest{}, errors.Join(errs...)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func decodeEntry(p string, v any) (IndexEntry, error) {
|
||||
var e IndexEntry
|
||||
errs := []error{checkCategoryPath(p)}
|
||||
fields, isObject := v.(map[string]any)
|
||||
if !isObject {
|
||||
return e, fmt.Errorf("an entry is a JSON object of parent, paths and reads")
|
||||
}
|
||||
if _, has := fields["paths"]; !has {
|
||||
errs = append(errs, fmt.Errorf("paths is missing"))
|
||||
}
|
||||
var err error
|
||||
for _, k := range sortedKeys(fields) {
|
||||
switch k {
|
||||
case "parent":
|
||||
var isString bool
|
||||
if e.Parent, isString = fields[k].(string); !isString {
|
||||
err = fmt.Errorf("parent is a string")
|
||||
} else if err = grammar.CheckIdentifier(e.Parent); err != nil {
|
||||
err = fmt.Errorf("parent %w", err)
|
||||
}
|
||||
case "paths":
|
||||
e.Paths, err = stringList(k, fields[k], func(q string) error {
|
||||
if q == "" {
|
||||
return nil
|
||||
}
|
||||
return checkCategoryPath(q)
|
||||
})
|
||||
case "reads":
|
||||
e.Reads, err = stringList(k, fields[k], checkCategoryPath)
|
||||
default:
|
||||
err = fmt.Errorf("unknown key %q; an entry holds parent, paths and reads", k)
|
||||
}
|
||||
errs = append(errs, err)
|
||||
}
|
||||
return e, errors.Join(errs...)
|
||||
}
|
||||
|
||||
// stringList is the JSON array of strings v, each passing check.
|
||||
func stringList(key string, v any, check func(string) error) ([]string, error) {
|
||||
items, isArray := v.([]any)
|
||||
if !isArray {
|
||||
return nil, fmt.Errorf("%s is an array of strings", key)
|
||||
}
|
||||
out := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
s, isString := item.(string)
|
||||
if !isString {
|
||||
return nil, fmt.Errorf("%s is an array of strings", key)
|
||||
}
|
||||
if err := check(s); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", key, err)
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// checkParent refuses a parent naming no entry beside p: a table loads with its parent,
|
||||
// from one source.
|
||||
func checkParent(index map[string]IndexEntry, p string) error {
|
||||
parent := index[p].Parent
|
||||
if parent == "" {
|
||||
return nil
|
||||
}
|
||||
sibling := parent
|
||||
if dot := strings.LastIndexByte(p, '.'); dot >= 0 {
|
||||
sibling = p[:dot+1] + parent
|
||||
}
|
||||
if _, ok := index[sibling]; !ok {
|
||||
return fmt.Errorf("parent %q names no entry beside it in this index", parent)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkCategoryPath(p string) error {
|
||||
for _, seg := range strings.Split(p, ".") {
|
||||
if err := grammar.CheckIdentifier(seg); err != nil {
|
||||
return fmt.Errorf("%q: %w", p, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
// Walk hands compile every category of the tree, folders and files in name order. A
|
||||
// folder holding no category anywhere below it is skipped; a hidden file or folder is no
|
||||
// category, though a category may name a hidden rows file beside it.
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
package datafiles
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/larvit/fejkdata/internal/grammar"
|
||||
"github.com/larvit/fejkdata/internal/jsonvalue"
|
||||
)
|
||||
|
||||
// ManifestFile is the file at a source's root that describes the source.
|
||||
const ManifestFile = ".fejkdata.json"
|
||||
|
||||
// Manifest is what a source's manifest says of it.
|
||||
type Manifest struct {
|
||||
// Index is every category of the source by dot path; nil where the manifest has none.
|
||||
Index map[string]IndexEntry `json:"index"`
|
||||
}
|
||||
|
||||
// IndexEntry is what an index says of a category: its parent table, "" for none, and the
|
||||
// paths List advertises below it.
|
||||
type IndexEntry struct {
|
||||
Parent string `json:"parent,omitempty"`
|
||||
Paths []string `json:"paths"`
|
||||
}
|
||||
|
||||
// Manifest reads the manifest at the source's root, the zero Manifest where there is none.
|
||||
func (s Source) Manifest() (Manifest, error) {
|
||||
if err := s.check(); err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
b, err := fs.ReadFile(s.fsys, path.Join(s.base, ManifestFile))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return Manifest{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err)
|
||||
}
|
||||
m, err := decodeManifest(b)
|
||||
if err != nil {
|
||||
return Manifest{}, fmt.Errorf("%s: %w", s.ManifestPath(), err)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// ManifestPath is the manifest's path as errors name it.
|
||||
func (s Source) ManifestPath() string { return s.labelled(path.Join(s.base, ManifestFile)) }
|
||||
|
||||
// decodeManifest decodes a manifest, refusing any key or shape it does not define, and
|
||||
// reports every mistake: first each entry's own, in path order, then each parent naming
|
||||
// no entry.
|
||||
func decodeManifest(b []byte) (Manifest, error) {
|
||||
raw, err := jsonvalue.Decode(b)
|
||||
if err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
top, isObject := raw.Any().(map[string]any)
|
||||
if !isObject {
|
||||
return Manifest{}, fmt.Errorf("a manifest must be an object, not %s", jsonvalue.Kind(raw.Any()))
|
||||
}
|
||||
var m Manifest
|
||||
var errs []error
|
||||
for _, k := range sortedKeys(top) {
|
||||
if k != "index" {
|
||||
errs = append(errs, fmt.Errorf("unknown key %q; a manifest holds index", k))
|
||||
continue
|
||||
}
|
||||
var indexErrs []error
|
||||
m.Index, indexErrs = decodeIndex(top[k])
|
||||
errs = append(errs, indexErrs...)
|
||||
}
|
||||
return m, errors.Join(errs...)
|
||||
}
|
||||
|
||||
func decodeIndex(v any) (map[string]IndexEntry, []error) {
|
||||
entries, isObject := v.(map[string]any)
|
||||
if !isObject {
|
||||
return nil, []error{fmt.Errorf("index must be an object, not %s", jsonvalue.Kind(v))}
|
||||
}
|
||||
index := map[string]IndexEntry{}
|
||||
var errs []error
|
||||
for _, p := range sortedKeys(entries) {
|
||||
e, err := decodeEntry(p, entries[p])
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("index entry %q: %w", p, err))
|
||||
}
|
||||
index[p] = e
|
||||
}
|
||||
for _, p := range sortedKeys(index) {
|
||||
if err := checkParent(index, p); err != nil {
|
||||
errs = append(errs, fmt.Errorf("index entry %q: %w", p, err))
|
||||
}
|
||||
}
|
||||
return index, errs
|
||||
}
|
||||
|
||||
// decodeEntry decodes one entry, leaving out a parent it refuses, so checkParent names no
|
||||
// mistake twice.
|
||||
func decodeEntry(p string, v any) (IndexEntry, error) {
|
||||
var e IndexEntry
|
||||
errs := []error{checkCategoryPath(p)}
|
||||
fields, isObject := v.(map[string]any)
|
||||
if !isObject {
|
||||
errs = append(errs, fmt.Errorf("an entry must be an object, not %s", jsonvalue.Kind(v)))
|
||||
return e, errors.Join(errs...)
|
||||
}
|
||||
if _, has := fields["paths"]; !has {
|
||||
errs = append(errs, fmt.Errorf("paths is missing"))
|
||||
}
|
||||
for _, k := range sortedKeys(fields) {
|
||||
switch k {
|
||||
case "parent":
|
||||
parent, err := decodeParent(fields[k])
|
||||
e.Parent = parent
|
||||
errs = append(errs, err)
|
||||
case "paths":
|
||||
var err error
|
||||
e.Paths, err = decodePaths(fields[k])
|
||||
errs = append(errs, err)
|
||||
default:
|
||||
errs = append(errs, fmt.Errorf("unknown key %q; an entry holds parent and paths", k))
|
||||
}
|
||||
}
|
||||
return e, errors.Join(errs...)
|
||||
}
|
||||
|
||||
func decodeParent(v any) (string, error) {
|
||||
parent, isString := v.(string)
|
||||
if !isString {
|
||||
return "", fmt.Errorf("parent must be a string, not %s", jsonvalue.Kind(v))
|
||||
}
|
||||
if err := grammar.CheckIdentifier(parent); err != nil {
|
||||
return "", fmt.Errorf("parent %w", err)
|
||||
}
|
||||
return parent, nil
|
||||
}
|
||||
|
||||
// decodePaths decodes a list of paths below a category, "" being the category itself.
|
||||
func decodePaths(v any) ([]string, error) {
|
||||
items, isList := v.([]any)
|
||||
if !isList {
|
||||
return nil, fmt.Errorf("paths must be a list, not %s", jsonvalue.Kind(v))
|
||||
}
|
||||
out := make([]string, 0, len(items))
|
||||
var errs []error
|
||||
for i, item := range items {
|
||||
q, isString := item.(string)
|
||||
if !isString {
|
||||
errs = append(errs, fmt.Errorf("paths item %d must be a string, not %s", i+1, jsonvalue.Kind(item)))
|
||||
continue
|
||||
}
|
||||
if q != "" {
|
||||
if err := checkCategoryPath(q); err != nil {
|
||||
errs = append(errs, fmt.Errorf("paths item %d: %w", i+1, err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, q)
|
||||
}
|
||||
return out, errors.Join(errs...)
|
||||
}
|
||||
|
||||
// checkParent refuses a parent naming no entry beside p: a table loads with its parent,
|
||||
// from one source.
|
||||
func checkParent(index map[string]IndexEntry, p string) error {
|
||||
parent := index[p].Parent
|
||||
if parent == "" {
|
||||
return nil
|
||||
}
|
||||
sibling := parent
|
||||
if dot := strings.LastIndexByte(p, '.'); dot >= 0 {
|
||||
sibling = p[:dot+1] + parent
|
||||
}
|
||||
if _, ok := index[sibling]; !ok {
|
||||
return fmt.Errorf("parent %q names no entry beside it in this index", parent)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkCategoryPath(p string) error {
|
||||
for _, seg := range strings.Split(p, ".") {
|
||||
if err := grammar.CheckIdentifier(seg); err != nil {
|
||||
return fmt.Errorf("%q: %w", p, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
@@ -4,6 +4,7 @@ package jsonvalue
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Value is a JSON value that only Decode, String and Object make, so a template never compiles
|
||||
@@ -37,3 +38,22 @@ func Object(fields map[string]Value) Value {
|
||||
// Any is the value as encoding/json decodes it: a map[string]any, []any, string, json.Number,
|
||||
// bool or nil.
|
||||
func (v Value) Any() any { return v.v }
|
||||
|
||||
// Kind names the kind of a value as Any gives it, such as "a number" or "a list".
|
||||
func Kind(v any) string {
|
||||
switch v.(type) {
|
||||
case json.Number:
|
||||
return "a number"
|
||||
case bool:
|
||||
return "a boolean"
|
||||
case string:
|
||||
return "a string"
|
||||
case []any:
|
||||
return "a list"
|
||||
case map[string]any:
|
||||
return "an object"
|
||||
case nil:
|
||||
return "null"
|
||||
}
|
||||
return fmt.Sprintf("%T", v)
|
||||
}
|
||||
|
||||
@@ -193,22 +193,6 @@ func typedOrText(text string, ds ...DataType) error {
|
||||
|
||||
// jsonKind names the kind of a JSON value an option cannot take, in the data format's own
|
||||
// terms rather than the decoding library's.
|
||||
func jsonKind(v any) string {
|
||||
switch v.(type) {
|
||||
case json.Number:
|
||||
return "a number"
|
||||
case bool:
|
||||
return "a boolean"
|
||||
case string:
|
||||
return "a string"
|
||||
case []any:
|
||||
return "a list"
|
||||
case map[string]any:
|
||||
return "an object"
|
||||
}
|
||||
return fmt.Sprintf("%T", v)
|
||||
}
|
||||
|
||||
func compileString(s string) (*template, error) {
|
||||
toks, unbound, err := parseChecked(s, nil)
|
||||
if err != nil {
|
||||
@@ -332,7 +316,7 @@ func readOptions(m map[string]any, pos position) (templateOptions, error) {
|
||||
}
|
||||
if sv, ok := m["separator"]; ok {
|
||||
if o.separator, ok = sv.(string); !ok {
|
||||
return o, fmt.Errorf("separator must be a string, not %s", jsonKind(sv))
|
||||
return o, fmt.Errorf("separator must be a string, not %s", jsonvalue.Kind(sv))
|
||||
}
|
||||
}
|
||||
return o, nil
|
||||
@@ -365,7 +349,7 @@ func repeatOf(m map[string]any) (int, error) {
|
||||
}
|
||||
r, ok := numberOf(rv)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("repeat must be a number, not %s", jsonKind(rv))
|
||||
return 0, fmt.Errorf("repeat must be a number, not %s", jsonvalue.Kind(rv))
|
||||
}
|
||||
if err := notTiny("repeat", rv, r); err != nil {
|
||||
return 0, err
|
||||
@@ -392,7 +376,7 @@ func weightOf(raw any) (float64, error) {
|
||||
}
|
||||
w, ok := numberOf(wv)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf(`weight is an option and takes a number, not %s; a field cannot be named "weight"`, jsonKind(wv))
|
||||
return 0, fmt.Errorf(`weight is an option and takes a number, not %s; a field cannot be named "weight"`, jsonvalue.Kind(wv))
|
||||
}
|
||||
if w < 0 || math.IsInf(w, 0) {
|
||||
return 0, fmt.Errorf("weight must be a finite number of 0 or more, got %v", wv)
|
||||
|
||||
+1
-1
@@ -177,7 +177,7 @@ func BenchmarkWideTokens500(b *testing.B) {
|
||||
|
||||
// New decodes the shipped manifest and loads nothing else.
|
||||
func TestNoNewAllocRegression(t *testing.T) {
|
||||
const base = 2501.0
|
||||
const base = 2449.0
|
||||
allocs := testing.AllocsPerRun(100, func() {
|
||||
if _, err := New(WithSeed(1)); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
+1
-1
@@ -263,7 +263,7 @@ func TestABrokenManifestFailsNewNamingEveryMistake(t *testing.T) {
|
||||
`{`: {datafiles.ManifestFile},
|
||||
`{"index": {}} {}`: {"after top-level value"},
|
||||
`{"indx": {}, "INDEX": {}}`: {`unknown key "INDEX"`, `unknown key "indx"`},
|
||||
`{"index": []}`: {"index must be an object, not an array"},
|
||||
`{"index": []}`: {"index must be an object, not a list"},
|
||||
`{"index": {"a..b": 5}}`: {`"a..b"`, "must be an object, not a number"},
|
||||
`{"index": {"a": {"Paths": [""]}}}`: {`unknown key "Paths"`, "paths is missing"},
|
||||
`{"index": {"a": {}, "b": {}}}`: {`"a": paths is missing`, `"b": paths is missing`},
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"github.com/larvit/fejkdata/internal/drawstate"
|
||||
"github.com/larvit/fejkdata/internal/grammar"
|
||||
"github.com/larvit/fejkdata/internal/invariant"
|
||||
"github.com/larvit/fejkdata/internal/jsonvalue"
|
||||
"github.com/larvit/fejkdata/internal/rows"
|
||||
)
|
||||
|
||||
@@ -102,7 +103,7 @@ func readTableOptions(m map[string]any) (tableOptionValues, error) {
|
||||
}
|
||||
s, isString := v.(string)
|
||||
if !isString {
|
||||
return o, fmt.Errorf("%s must be a string, not %s", k, jsonKind(v))
|
||||
return o, fmt.Errorf("%s must be a string, not %s", k, jsonvalue.Kind(v))
|
||||
}
|
||||
if k != "format" && s == "" {
|
||||
return o, fmt.Errorf("%s names a column, so it cannot be empty; drop it", k)
|
||||
|
||||
Reference in New Issue
Block a user