From 818a9b56c2be109ff1807192dd928806ab3321a9 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Mon, 5 Oct 2026 15:32:39 +0200 Subject: [PATCH] Move the folder walk and the reading of a data tree's JSON and TSV files into internal/datafiles --- README.md | 3 +- data.go | 160 ++++------------------------ fejkdata.go | 9 +- internal/datafiles/datafiles.go | 178 ++++++++++++++++++++++++++++++++ node.go | 4 +- shipped.go | 29 ++---- table.go | 4 +- 7 files changed, 218 insertions(+), 169 deletions(-) create mode 100644 internal/datafiles/datafiles.go diff --git a/README.md b/README.md index 242bb4f..59a81f5 100644 --- a/README.md +++ b/README.md @@ -971,9 +971,10 @@ graph.go the node graph: tree walks, containment and render edges, and th calc.go the {calc()} operand checks: never a number, and a constant zero divisor datatype.go column datatypes: DataType, where datatype and null may sit, a column's datatype valueproof.go the value proof: what a typed column, calc operand or struct field holds, checked at load -data.go the load path: fs.FS folders/files -> merged namespace tree, and the bind pipeline every entry point runs +data.go the load path: the categories internal/datafiles hands over, compiled into one merged tree, and the bind pipeline every entry point runs cmd/fejkdata/ the fejkdata CLI internal/builtinfunc/ the {name()} functions: their checks and draws, checksums, date and time layouts, transforms and calc arithmetic +internal/datafiles/ the walk of a data tree: its folders, its category files and the rows files beside them internal/datatype/ what a record column holds, which DataType aliases internal/drawstate/ the seeded randomness and the {seq()} counters a generator draws through internal/grammar/ how the template language is written: format tokens, paths and selectors, reference sigils, names, calc syntax, and whether an argument is a template or a path diff --git a/data.go b/data.go index 035cc87..3ec2993 100644 --- a/data.go +++ b/data.go @@ -1,55 +1,17 @@ package fejkdata import ( - "encoding/json" "fmt" - "io/fs" - "os" - "path" "strings" - "github.com/larvit/fejkdata/internal/grammar" + "github.com/larvit/fejkdata/internal/datafiles" ) -// dataSource is one tree to load: an fs.FS and the directory in it to start from. -// label prefixes file names in errors; onDisk marks diskPath as a directory that must -// exist. -type dataSource struct { - fsys fs.FS - label string - onDisk bool - diskPath string - baseDir string -} - -func (s dataSource) labelled(p string) string { - if s.label == "" { - return p - } - return path.Join(s.label, p) -} - -func loadData(sources []dataSource) (map[string]node, error) { +func loadData(sources []datafiles.Source) (map[string]node, error) { root := map[string]node{} for _, src := range sources { - if src.onDisk { - if src.diskPath == "" { - return nil, fmt.Errorf("a data path is empty") - } - info, err := os.Stat(src.diskPath) - if err != nil { - return nil, fmt.Errorf("data path %s: %w", src.diskPath, err) - } - if !info.IsDir() { - return nil, fmt.Errorf("%s is not a directory", src.diskPath) - } - } - dir := src.baseDir - if dir == "" { - dir = "." - } - g, err := loadDir(src, dir) - if err != nil { + g := &folder{children: map[string]node{}} + if err := src.Walk(compileInto(g)); err != nil { return nil, err } mergeChildren(root, g.children) @@ -63,108 +25,30 @@ func loadData(sources []dataSource) (map[string]node, error) { return root, nil } -// loadDir compiles one directory into a folder. Empty subdirectories (no JSON -// anywhere under them) are skipped rather than added as empty namespaces. -func loadDir(src dataSource, dir string) (*folder, error) { - entries, err := fs.ReadDir(src.fsys, dir) - if err != nil { - return nil, fmt.Errorf("%s: %w", src.labelled(dir), err) - } - g := &folder{children: map[string]node{}} - files := newCategoryFiles(src, dir, entries) - for _, e := range entries { - if strings.HasPrefix(e.Name(), ".") { // hidden: a checkout or an editor's file, never data - continue +// compileInto compiles each category it is handed into its folder under root. +func compileInto(root *folder) func(datafiles.Category) error { + return func(c datafiles.Category) error { + n, err := compileCategory(c.JSON, c.Name, c.ReadRows) + if err != nil { + return err } - full := path.Join(dir, e.Name()) - if e.IsDir() { - if err := loadFolder(src, g, full, e.Name()); err != nil { - return nil, err - } - continue - } - if err := loadFile(src, g, full, e.Name(), files); err != nil { - return nil, err - } - } - for name, named := range files.tsv { - if !named && !strings.HasPrefix(name, ".") { - return nil, fmt.Errorf("%s: no category names it in its rows; a table's rows file sits beside a category file naming it", src.labelled(path.Join(dir, name))) - } - } - return g, nil -} - -// categoryFiles is what a category may name beside itself: the rows files of its -// directory, each marked once a category names it. -type categoryFiles struct { - src dataSource - dir string - tsv map[string]bool -} - -func newCategoryFiles(src dataSource, dir string, entries []fs.DirEntry) *categoryFiles { - files := &categoryFiles{src: src, dir: dir, tsv: map[string]bool{}} - for _, e := range entries { - if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() { - files.tsv[e.Name()] = false - } - } - return files -} - -func (c *categoryFiles) readRows(name string) (string, error) { - if _, present := c.tsv[name]; !present { - return "", fmt.Errorf("rows names %s, which is not beside it in %s", name, c.src.labelled(c.dir)) - } - c.tsv[name] = true - b, err := fs.ReadFile(c.src.fsys, path.Join(c.dir, name)) - if err != nil { - return "", fmt.Errorf("%s: %w", c.src.labelled(path.Join(c.dir, name)), err) - } - return string(b), nil -} - -// loadFolder adds a subdirectory as a nested folder, unless nothing under it is data. -func loadFolder(src dataSource, g *folder, full, name string) error { - child, err := loadDir(src, full) - if err != nil { - return err - } - if len(child.children) == 0 { + madeFolder(root, c.Dir).children[c.Name] = n return nil } - if err := grammar.CheckName(name); err != nil { - return fmt.Errorf("%s: folder %w", src.labelled(full), err) - } - g.children[name] = child - return nil } -// loadFile compiles a *.json file into a category named after it; any other file -// is skipped. -func loadFile(src dataSource, g *folder, full, file string, files *categoryFiles) error { - if !strings.HasSuffix(file, ".json") { - return nil +// madeFolder is the folder dir names under root, made where it is missing or a category. +func madeFolder(root *folder, dir []string) *folder { + g := root + for _, seg := range dir { + sub, isFolder := g.children[seg].(*folder) + if !isFolder { + sub = &folder{children: map[string]node{}} + g.children[seg] = sub + } + g = sub } - name := strings.TrimSuffix(file, ".json") - if err := grammar.CheckName(name); err != nil { - return fmt.Errorf("%s: category %w", src.labelled(full), err) - } - b, err := fs.ReadFile(src.fsys, full) - if err != nil { - return fmt.Errorf("%s: %w", src.labelled(full), err) - } - var raw any - if err := json.Unmarshal(b, &raw); err != nil { - return fmt.Errorf("%s: %w", src.labelled(full), err) - } - n, err := compileCategory(raw, name, files) - if err != nil { - return fmt.Errorf("%s: %w", src.labelled(full), err) - } - g.children[name] = n - return nil + return g } // mergeChildren overlays src onto dst. Two folders under the same key merge diff --git a/fejkdata.go b/fejkdata.go index effd449..c6c9d95 100644 --- a/fejkdata.go +++ b/fejkdata.go @@ -11,6 +11,7 @@ import ( "sort" "sync" + "github.com/larvit/fejkdata/internal/datafiles" "github.com/larvit/fejkdata/internal/drawstate" ) @@ -40,7 +41,7 @@ type config struct { seed uint64 seeded bool shipped bool - sources []dataSource + sources []datafiles.Source } // Option configures a [Generator]. @@ -56,13 +57,13 @@ func WithSeed(seed uint64) Option { // layer several; the last wins a name clash. func WithDataPath(dir string) Option { return func(c *config) { - c.sources = append(c.sources, dataSource{fsys: os.DirFS(dir), label: dir, onDisk: true, diskPath: dir}) + c.sources = append(c.sources, datafiles.Source{FS: os.DirFS(dir), Label: dir, OnDisk: true}) } } // WithDataFS layers a data tree held in an [fs.FS], such as an embed.FS of your own. func WithDataFS(fsys fs.FS) Option { - return func(c *config) { c.sources = append(c.sources, dataSource{fsys: fsys}) } + return func(c *config) { c.sources = append(c.sources, datafiles.Source{FS: fsys}) } } // WithoutShippedData leaves the shipped data set out, so only the sources given @@ -99,7 +100,7 @@ func (c config) load() (folder, error) { if c.shipped && len(c.sources) == 0 { return unloadedTree(), nil } - var sources []dataSource + var sources []datafiles.Source if c.shipped { sources = append(sources, shippedSource) } diff --git a/internal/datafiles/datafiles.go b/internal/datafiles/datafiles.go new file mode 100644 index 0000000..e1070cb --- /dev/null +++ b/internal/datafiles/datafiles.go @@ -0,0 +1,178 @@ +// Package datafiles walks a data tree and reads its category files and their rows. +package datafiles + +import ( + "encoding/json" + "fmt" + "io/fs" + "os" + "path" + "strings" + + "github.com/larvit/fejkdata/internal/grammar" +) + +// Source is one tree to load: an fs.FS and the directory in it to start from. Label +// prefixes file names in errors; OnDisk marks Label as a directory that must exist. +type Source struct { + FS fs.FS + Label string + OnDisk bool + BaseDir string +} + +// Category is one category file: the folders above it from the source's base, its name, +// and its parsed JSON. +type Category struct { + Dir []string + Name string + JSON any + rows *rowsFiles +} + +// ReadRows reads a rows file beside the category, marking it named. +func (c Category) ReadRows(name string) (string, error) { return c.rows.read(name) } + +func (s Source) labelled(p string) string { + if s.Label == "" { + return p + } + return path.Join(s.Label, p) +} + +func (s Source) base() string { + if s.BaseDir == "" { + return "." + } + return s.BaseDir +} + +// Walk hands compile every category of the tree, folders and files in name order. A +// folder holding no category anywhere below it is skipped; a hidden file or folder is +// never data. +func (s Source) Walk(compile func(Category) error) error { + if s.OnDisk { + if s.Label == "" { + return fmt.Errorf("a data path is empty") + } + info, err := os.Stat(s.Label) + if err != nil { + return fmt.Errorf("data path %s: %w", s.Label, err) + } + if !info.IsDir() { + return fmt.Errorf("%s is not a directory", s.Label) + } + } + _, err := s.walkDir(nil, compile) + return err +} + +// walkDir walks the folder dir names, and reports whether it handed over any category. +func (s Source) walkDir(dir []string, compile func(Category) error) (bool, error) { + full := path.Join(append([]string{s.base()}, dir...)...) + entries, err := fs.ReadDir(s.FS, full) + if err != nil { + return false, fmt.Errorf("%s: %w", s.labelled(full), err) + } + rows := newRowsFiles(s, full, entries) + handed := false + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".") { // hidden: a checkout or an editor's file, never data + continue + } + var holds bool + if e.IsDir() { + holds, err = s.walkFolder(append(dir[:len(dir):len(dir)], e.Name()), compile) + } else { + holds, err = s.compileFile(dir, e.Name(), rows, compile) + } + if err != nil { + return false, err + } + handed = handed || holds + } + for name, named := range rows.tsv { + if !named && !strings.HasPrefix(name, ".") { + return false, fmt.Errorf("%s: no category names it in its rows; a table's rows file sits beside a category file naming it", s.labelled(path.Join(full, name))) + } + } + return handed, nil +} + +// walkFolder walks a subfolder, and refuses its name once it holds a category. +func (s Source) walkFolder(dir []string, compile func(Category) error) (bool, error) { + holds, err := s.walkDir(dir, compile) + if err != nil || !holds { + return false, err + } + if err := grammar.CheckName(dir[len(dir)-1]); err != nil { + return false, fmt.Errorf("%s: folder %w", s.labelled(path.Join(append([]string{s.base()}, dir...)...)), err) + } + return true, nil +} + +// Load hands compile the one category file name in the folder dir. +func (s Source) Load(dir []string, name string, compile func(Category) error) error { + full := path.Join(append([]string{s.base()}, dir...)...) + entries, err := fs.ReadDir(s.FS, full) + if err != nil { + return fmt.Errorf("%s: %w", s.labelled(full), err) + } + _, err = s.compileFile(dir, name+".json", newRowsFiles(s, full, entries), compile) + return err +} + +// compileFile hands compile a *.json file as a category named after it, and reports +// whether it was one; any other file is skipped. +func (s Source) compileFile(dir []string, file string, rows *rowsFiles, compile func(Category) error) (bool, error) { + if !strings.HasSuffix(file, ".json") { + return false, nil + } + full := path.Join(rows.dir, file) + name := strings.TrimSuffix(file, ".json") + if err := grammar.CheckName(name); err != nil { + return false, fmt.Errorf("%s: category %w", s.labelled(full), err) + } + b, err := fs.ReadFile(s.FS, full) + if err != nil { + return false, fmt.Errorf("%s: %w", s.labelled(full), err) + } + var raw any + if err := json.Unmarshal(b, &raw); err != nil { + return false, fmt.Errorf("%s: %w", s.labelled(full), err) + } + if err := compile(Category{Dir: dir, Name: name, JSON: raw, rows: rows}); err != nil { + return false, fmt.Errorf("%s: %w", s.labelled(full), err) + } + return true, nil +} + +// rowsFiles is what a category may name beside itself: the rows files of its folder, +// each marked once a category names it. +type rowsFiles struct { + src Source + dir string + tsv map[string]bool +} + +func newRowsFiles(src Source, dir string, entries []fs.DirEntry) *rowsFiles { + files := &rowsFiles{src: src, dir: dir, tsv: map[string]bool{}} + for _, e := range entries { + if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() { + files.tsv[e.Name()] = false + } + } + return files +} + +func (r *rowsFiles) read(name string) (string, error) { + if _, present := r.tsv[name]; !present { + return "", fmt.Errorf("rows names %s, which is not beside it in %s", name, r.src.labelled(r.dir)) + } + r.tsv[name] = true + b, err := fs.ReadFile(r.src.FS, path.Join(r.dir, name)) + if err != nil { + return "", fmt.Errorf("%s: %w", r.src.labelled(path.Join(r.dir, name)), err) + } + return string(b), nil +} diff --git a/node.go b/node.go index daf8e4c..ea1e66f 100644 --- a/node.go +++ b/node.go @@ -119,10 +119,10 @@ func compile(v any) (node, error) { // compileCategory compiles a data file's value, which may be a table over a rows // file beside it, and refuses a choice that is a table written as templates. -func compileCategory(v any, name string, files *categoryFiles) (node, error) { +func compileCategory(v any, name string, readRows func(file string) (string, error)) (node, error) { if m, ok := v.(map[string]any); ok { if _, isTable := m["rows"]; isTable { - return compileTable(m, name, files) + return compileTable(m, name, readRows) } } if items, ok := v.([]any); ok { diff --git a/shipped.go b/shipped.go index b8aafd7..f363510 100644 --- a/shipped.go +++ b/shipped.go @@ -2,10 +2,9 @@ package fejkdata import ( "embed" - "fmt" - "io/fs" - "path" "strings" + + "github.com/larvit/fejkdata/internal/datafiles" ) //go:generate env REGENERATE=1 go test -run ^TestShippedIndexIsCurrent$ . @@ -13,7 +12,7 @@ import ( //go:embed data var shippedFS embed.FS -var shippedSource = dataSource{fsys: shippedFS, baseDir: "data"} +var shippedSource = datafiles.Source{FS: shippedFS, BaseDir: "data"} // shippedEntry is a shipped category's entry in shippedindex.go: its parent table, "" for // none, and the paths List advertises below it. After changing it, empty @@ -28,15 +27,7 @@ func unloadedTree() folder { root := folder{children: map[string]node{}} for p, e := range shippedIndex { segs := strings.Split(p, ".") - g := &root - for _, seg := range segs[:len(segs)-1] { - sub, isFolder := g.children[seg].(*folder) - if !isFolder { - sub = &folder{children: map[string]node{}} - g.children[seg] = sub - } - g = sub - } + g := madeFolder(&root, segs[:len(segs)-1]) if g.unloaded == nil { g.unloaded = map[string]shippedEntry{} } @@ -107,7 +98,7 @@ func loadShipped(root *folder, wanted []unloadedCategory) { if !unloaded { continue } - site, err := u.load() + site, err := u.load(root) if err != nil { panic(internalError("shipped %s: %v; after a change under data/, regenerate shippedindex.go", join(strings.Join(u.dir, "."), u.name), err)) } @@ -125,14 +116,8 @@ func loadShipped(root *folder, wanted []unloadedCategory) { // load parses and compiles the category, and moves it from its folder's unloaded map to // the folder's children. -func (u unloadedCategory) load() (categorySite, error) { - dir := path.Join(append([]string{shippedSource.baseDir}, u.dir...)...) - entries, err := fs.ReadDir(shippedSource.fsys, dir) - if err != nil { - return categorySite{}, fmt.Errorf("%s: %w", dir, err) - } - file := u.name + ".json" - if err := loadFile(shippedSource, u.in, path.Join(dir, file), file, newCategoryFiles(shippedSource, dir, entries)); err != nil { +func (u unloadedCategory) load(root *folder) (categorySite, error) { + if err := shippedSource.Load(u.dir, u.name, compileInto(root)); err != nil { return categorySite{}, err } delete(u.in.unloaded, u.name) diff --git a/table.go b/table.go index 36b678c..26cc2ba 100644 --- a/table.go +++ b/table.go @@ -85,12 +85,12 @@ func isTableOption(name string) bool { // token arm. const inSelector = `[]{}"|` -func compileTable(m map[string]any, segment string, files *categoryFiles) (*table, error) { +func compileTable(m map[string]any, segment string, readRows func(file string) (string, error)) (*table, error) { o, err := readTableOptions(m) if err != nil { return nil, err } - data, err := files.readRows(o.rows) + data, err := readRows(o.rows) if err != nil { return nil, err }