Move the folder walk and the reading of a data tree's JSON and TSV files into internal/datafiles

README.md

@@ -971,9 +971,10 @@ graph.go        the node graph: tree walks, containment and render edges, and th
 calc.go         the {calc()} operand checks: never a number, and a constant zero divisor
 datatype.go     column datatypes: DataType, where datatype and null may sit, a column's datatype
 valueproof.go   the value proof: what a typed column, calc operand or struct field holds, checked at load
-data.go         the load path: fs.FS folders/files -> merged namespace tree, and the bind pipeline every entry point runs
+data.go         the load path: the categories internal/datafiles hands over, compiled into one merged tree, and the bind pipeline every entry point runs
 cmd/fejkdata/   the fejkdata CLI
 internal/builtinfunc/ the {name()} functions: their checks and draws, checksums, date and time layouts, transforms and calc arithmetic
+internal/datafiles/ the walk of a data tree: its folders, its category files and the rows files beside them
 internal/datatype/ what a record column holds, which DataType aliases
 internal/drawstate/ the seeded randomness and the {seq()} counters a generator draws through
 internal/grammar/ how the template language is written: format tokens, paths and selectors, reference sigils, names, calc syntax, and whether an argument is a template or a path

data.go

@@ -1,55 +1,17 @@
 package fejkdata
 
 import (
-	"encoding/json"
 	"fmt"
-	"io/fs"
-	"os"
-	"path"
 	"strings"
 
-	"github.com/larvit/fejkdata/internal/grammar"
+	"github.com/larvit/fejkdata/internal/datafiles"
 )
 
-// dataSource is one tree to load: an fs.FS and the directory in it to start from.
+func loadData(sources []datafiles.Source) (map[string]node, error) {
-// label prefixes file names in errors; onDisk marks diskPath as a directory that must
-// exist.
-type dataSource struct {
-	fsys     fs.FS
-	label    string
-	onDisk   bool
-	diskPath string
-	baseDir  string
-}
-
-func (s dataSource) labelled(p string) string {
-	if s.label == "" {
-		return p
-	}
-	return path.Join(s.label, p)
-}
-
-func loadData(sources []dataSource) (map[string]node, error) {
 	root := map[string]node{}
 	for _, src := range sources {
-		if src.onDisk {
+		g := &folder{children: map[string]node{}}
-			if src.diskPath == "" {
+		if err := src.Walk(compileInto(g)); err != nil {
-				return nil, fmt.Errorf("a data path is empty")
-			}
-			info, err := os.Stat(src.diskPath)
-			if err != nil {
-				return nil, fmt.Errorf("data path %s: %w", src.diskPath, err)
-			}
-			if !info.IsDir() {
-				return nil, fmt.Errorf("%s is not a directory", src.diskPath)
-			}
-		}
-		dir := src.baseDir
-		if dir == "" {
-			dir = "."
-		}
-		g, err := loadDir(src, dir)
-		if err != nil {
 			return nil, err
 		}
 		mergeChildren(root, g.children)
@@ -63,108 +25,30 @@ func loadData(sources []dataSource) (map[string]node, error) {
 	return root, nil
 }
 
-// loadDir compiles one directory into a folder. Empty subdirectories (no JSON
+// compileInto compiles each category it is handed into its folder under root.
-// anywhere under them) are skipped rather than added as empty namespaces.
+func compileInto(root *folder) func(datafiles.Category) error {
-func loadDir(src dataSource, dir string) (*folder, error) {
+	return func(c datafiles.Category) error {
-	entries, err := fs.ReadDir(src.fsys, dir)
+		n, err := compileCategory(c.JSON, c.Name, c.ReadRows)
-	if err != nil {
+		if err != nil {
-		return nil, fmt.Errorf("%s: %w", src.labelled(dir), err)
+			return err
-	}
-	g := &folder{children: map[string]node{}}
-	files := newCategoryFiles(src, dir, entries)
-	for _, e := range entries {
-		if strings.HasPrefix(e.Name(), ".") { // hidden: a checkout or an editor's file, never data
-			continue
 		}
-		full := path.Join(dir, e.Name())
+		madeFolder(root, c.Dir).children[c.Name] = n
-		if e.IsDir() {
-			if err := loadFolder(src, g, full, e.Name()); err != nil {
-				return nil, err
-			}
-			continue
-		}
-		if err := loadFile(src, g, full, e.Name(), files); err != nil {
-			return nil, err
-		}
-	}
-	for name, named := range files.tsv {
-		if !named && !strings.HasPrefix(name, ".") {
-			return nil, fmt.Errorf("%s: no category names it in its rows; a table's rows file sits beside a category file naming it", src.labelled(path.Join(dir, name)))
-		}
-	}
-	return g, nil
-}
-
-// categoryFiles is what a category may name beside itself: the rows files of its
-// directory, each marked once a category names it.
-type categoryFiles struct {
-	src dataSource
-	dir string
-	tsv map[string]bool
-}
-
-func newCategoryFiles(src dataSource, dir string, entries []fs.DirEntry) *categoryFiles {
-	files := &categoryFiles{src: src, dir: dir, tsv: map[string]bool{}}
-	for _, e := range entries {
-		if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() {
-			files.tsv[e.Name()] = false
-		}
-	}
-	return files
-}
-
-func (c *categoryFiles) readRows(name string) (string, error) {
-	if _, present := c.tsv[name]; !present {
-		return "", fmt.Errorf("rows names %s, which is not beside it in %s", name, c.src.labelled(c.dir))
-	}
-	c.tsv[name] = true
-	b, err := fs.ReadFile(c.src.fsys, path.Join(c.dir, name))
-	if err != nil {
-		return "", fmt.Errorf("%s: %w", c.src.labelled(path.Join(c.dir, name)), err)
-	}
-	return string(b), nil
-}
-
-// loadFolder adds a subdirectory as a nested folder, unless nothing under it is data.
-func loadFolder(src dataSource, g *folder, full, name string) error {
-	child, err := loadDir(src, full)
-	if err != nil {
-		return err
-	}
-	if len(child.children) == 0 {
 		return nil
 	}
-	if err := grammar.CheckName(name); err != nil {
-		return fmt.Errorf("%s: folder %w", src.labelled(full), err)
-	}
-	g.children[name] = child
-	return nil
 }
 
-// loadFile compiles a *.json file into a category named after it; any other file
+// madeFolder is the folder dir names under root, made where it is missing or a category.
-// is skipped.
+func madeFolder(root *folder, dir []string) *folder {
-func loadFile(src dataSource, g *folder, full, file string, files *categoryFiles) error {
+	g := root
-	if !strings.HasSuffix(file, ".json") {
+	for _, seg := range dir {
-		return nil
+		sub, isFolder := g.children[seg].(*folder)
+		if !isFolder {
+			sub = &folder{children: map[string]node{}}
+			g.children[seg] = sub
+		}
+		g = sub
 	}
-	name := strings.TrimSuffix(file, ".json")
+	return g
-	if err := grammar.CheckName(name); err != nil {
-		return fmt.Errorf("%s: category %w", src.labelled(full), err)
-	}
-	b, err := fs.ReadFile(src.fsys, full)
-	if err != nil {
-		return fmt.Errorf("%s: %w", src.labelled(full), err)
-	}
-	var raw any
-	if err := json.Unmarshal(b, &raw); err != nil {
-		return fmt.Errorf("%s: %w", src.labelled(full), err)
-	}
-	n, err := compileCategory(raw, name, files)
-	if err != nil {
-		return fmt.Errorf("%s: %w", src.labelled(full), err)
-	}
-	g.children[name] = n
-	return nil
 }
 
 // mergeChildren overlays src onto dst. Two folders under the same key merge

fejkdata.go

@@ -11,6 +11,7 @@ import (
 	"sort"
 	"sync"
 
+	"github.com/larvit/fejkdata/internal/datafiles"
 	"github.com/larvit/fejkdata/internal/drawstate"
 )
 
@@ -40,7 +41,7 @@ type config struct {
 	seed    uint64
 	seeded  bool
 	shipped bool
-	sources []dataSource
+	sources []datafiles.Source
 }
 
 // Option configures a [Generator].
@@ -56,13 +57,13 @@ func WithSeed(seed uint64) Option {
 // layer several; the last wins a name clash.
 func WithDataPath(dir string) Option {
 	return func(c *config) {
-		c.sources = append(c.sources, dataSource{fsys: os.DirFS(dir), label: dir, onDisk: true, diskPath: dir})
+		c.sources = append(c.sources, datafiles.Source{FS: os.DirFS(dir), Label: dir, OnDisk: true})
 	}
 }
 
 // WithDataFS layers a data tree held in an [fs.FS], such as an embed.FS of your own.
 func WithDataFS(fsys fs.FS) Option {
-	return func(c *config) { c.sources = append(c.sources, dataSource{fsys: fsys}) }
+	return func(c *config) { c.sources = append(c.sources, datafiles.Source{FS: fsys}) }
 }
 
 // WithoutShippedData leaves the shipped data set out, so only the sources given
@@ -99,7 +100,7 @@ func (c config) load() (folder, error) {
 	if c.shipped && len(c.sources) == 0 {
 		return unloadedTree(), nil
 	}
-	var sources []dataSource
+	var sources []datafiles.Source
 	if c.shipped {
 		sources = append(sources, shippedSource)
 	}

internal/datafiles/datafiles.go

@@ -0,0 +1,178 @@
+// Package datafiles walks a data tree and reads its category files and their rows.
+package datafiles
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/larvit/fejkdata/internal/grammar"
+)
+
+// Source is one tree to load: an fs.FS and the directory in it to start from. Label
+// prefixes file names in errors; OnDisk marks Label as a directory that must exist.
+type Source struct {
+	FS      fs.FS
+	Label   string
+	OnDisk  bool
+	BaseDir string
+}
+
+// Category is one category file: the folders above it from the source's base, its name,
+// and its parsed JSON.
+type Category struct {
+	Dir  []string
+	Name string
+	JSON any
+	rows *rowsFiles
+}
+
+// ReadRows reads a rows file beside the category, marking it named.
+func (c Category) ReadRows(name string) (string, error) { return c.rows.read(name) }
+
+func (s Source) labelled(p string) string {
+	if s.Label == "" {
+		return p
+	}
+	return path.Join(s.Label, p)
+}
+
+func (s Source) base() string {
+	if s.BaseDir == "" {
+		return "."
+	}
+	return s.BaseDir
+}
+
+// Walk hands compile every category of the tree, folders and files in name order. A
+// folder holding no category anywhere below it is skipped; a hidden file or folder is
+// never data.
+func (s Source) Walk(compile func(Category) error) error {
+	if s.OnDisk {
+		if s.Label == "" {
+			return fmt.Errorf("a data path is empty")
+		}
+		info, err := os.Stat(s.Label)
+		if err != nil {
+			return fmt.Errorf("data path %s: %w", s.Label, err)
+		}
+		if !info.IsDir() {
+			return fmt.Errorf("%s is not a directory", s.Label)
+		}
+	}
+	_, err := s.walkDir(nil, compile)
+	return err
+}
+
+// walkDir walks the folder dir names, and reports whether it handed over any category.
+func (s Source) walkDir(dir []string, compile func(Category) error) (bool, error) {
+	full := path.Join(append([]string{s.base()}, dir...)...)
+	entries, err := fs.ReadDir(s.FS, full)
+	if err != nil {
+		return false, fmt.Errorf("%s: %w", s.labelled(full), err)
+	}
+	rows := newRowsFiles(s, full, entries)
+	handed := false
+	for _, e := range entries {
+		if strings.HasPrefix(e.Name(), ".") { // hidden: a checkout or an editor's file, never data
+			continue
+		}
+		var holds bool
+		if e.IsDir() {
+			holds, err = s.walkFolder(append(dir[:len(dir):len(dir)], e.Name()), compile)
+		} else {
+			holds, err = s.compileFile(dir, e.Name(), rows, compile)
+		}
+		if err != nil {
+			return false, err
+		}
+		handed = handed || holds
+	}
+	for name, named := range rows.tsv {
+		if !named && !strings.HasPrefix(name, ".") {
+			return false, fmt.Errorf("%s: no category names it in its rows; a table's rows file sits beside a category file naming it", s.labelled(path.Join(full, name)))
+		}
+	}
+	return handed, nil
+}
+
+// walkFolder walks a subfolder, and refuses its name once it holds a category.
+func (s Source) walkFolder(dir []string, compile func(Category) error) (bool, error) {
+	holds, err := s.walkDir(dir, compile)
+	if err != nil || !holds {
+		return false, err
+	}
+	if err := grammar.CheckName(dir[len(dir)-1]); err != nil {
+		return false, fmt.Errorf("%s: folder %w", s.labelled(path.Join(append([]string{s.base()}, dir...)...)), err)
+	}
+	return true, nil
+}
+
+// Load hands compile the one category file name in the folder dir.
+func (s Source) Load(dir []string, name string, compile func(Category) error) error {
+	full := path.Join(append([]string{s.base()}, dir...)...)
+	entries, err := fs.ReadDir(s.FS, full)
+	if err != nil {
+		return fmt.Errorf("%s: %w", s.labelled(full), err)
+	}
+	_, err = s.compileFile(dir, name+".json", newRowsFiles(s, full, entries), compile)
+	return err
+}
+
+// compileFile hands compile a *.json file as a category named after it, and reports
+// whether it was one; any other file is skipped.
+func (s Source) compileFile(dir []string, file string, rows *rowsFiles, compile func(Category) error) (bool, error) {
+	if !strings.HasSuffix(file, ".json") {
+		return false, nil
+	}
+	full := path.Join(rows.dir, file)
+	name := strings.TrimSuffix(file, ".json")
+	if err := grammar.CheckName(name); err != nil {
+		return false, fmt.Errorf("%s: category %w", s.labelled(full), err)
+	}
+	b, err := fs.ReadFile(s.FS, full)
+	if err != nil {
+		return false, fmt.Errorf("%s: %w", s.labelled(full), err)
+	}
+	var raw any
+	if err := json.Unmarshal(b, &raw); err != nil {
+		return false, fmt.Errorf("%s: %w", s.labelled(full), err)
+	}
+	if err := compile(Category{Dir: dir, Name: name, JSON: raw, rows: rows}); err != nil {
+		return false, fmt.Errorf("%s: %w", s.labelled(full), err)
+	}
+	return true, nil
+}
+
+// rowsFiles is what a category may name beside itself: the rows files of its folder,
+// each marked once a category names it.
+type rowsFiles struct {
+	src Source
+	dir string
+	tsv map[string]bool
+}
+
+func newRowsFiles(src Source, dir string, entries []fs.DirEntry) *rowsFiles {
+	files := &rowsFiles{src: src, dir: dir, tsv: map[string]bool{}}
+	for _, e := range entries {
+		if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() {
+			files.tsv[e.Name()] = false
+		}
+	}
+	return files
+}
+
+func (r *rowsFiles) read(name string) (string, error) {
+	if _, present := r.tsv[name]; !present {
+		return "", fmt.Errorf("rows names %s, which is not beside it in %s", name, r.src.labelled(r.dir))
+	}
+	r.tsv[name] = true
+	b, err := fs.ReadFile(r.src.FS, path.Join(r.dir, name))
+	if err != nil {
+		return "", fmt.Errorf("%s: %w", r.src.labelled(path.Join(r.dir, name)), err)
+	}
+	return string(b), nil
+}

node.go

@@ -119,10 +119,10 @@ func compile(v any) (node, error) {
 
 // compileCategory compiles a data file's value, which may be a table over a rows
 // file beside it, and refuses a choice that is a table written as templates.
-func compileCategory(v any, name string, files *categoryFiles) (node, error) {
+func compileCategory(v any, name string, readRows func(file string) (string, error)) (node, error) {
 	if m, ok := v.(map[string]any); ok {
 		if _, isTable := m["rows"]; isTable {
-			return compileTable(m, name, files)
+			return compileTable(m, name, readRows)
 		}
 	}
 	if items, ok := v.([]any); ok {

shipped.go

@@ -2,10 +2,9 @@ package fejkdata
 
 import (
 	"embed"
-	"fmt"
-	"io/fs"
-	"path"
 	"strings"
+
+	"github.com/larvit/fejkdata/internal/datafiles"
 )
 
 //go:generate env REGENERATE=1 go test -run ^TestShippedIndexIsCurrent$ .
@@ -13,7 +12,7 @@ import (
 //go:embed data
 var shippedFS embed.FS
 
-var shippedSource = dataSource{fsys: shippedFS, baseDir: "data"}
+var shippedSource = datafiles.Source{FS: shippedFS, BaseDir: "data"}
 
 // shippedEntry is a shipped category's entry in shippedindex.go: its parent table, "" for
 // none, and the paths List advertises below it. After changing it, empty
@@ -28,15 +27,7 @@ func unloadedTree() folder {
 	root := folder{children: map[string]node{}}
 	for p, e := range shippedIndex {
 		segs := strings.Split(p, ".")
-		g := &root
+		g := madeFolder(&root, segs[:len(segs)-1])
-		for _, seg := range segs[:len(segs)-1] {
-			sub, isFolder := g.children[seg].(*folder)
-			if !isFolder {
-				sub = &folder{children: map[string]node{}}
-				g.children[seg] = sub
-			}
-			g = sub
-		}
 		if g.unloaded == nil {
 			g.unloaded = map[string]shippedEntry{}
 		}
@@ -107,7 +98,7 @@ func loadShipped(root *folder, wanted []unloadedCategory) {
 		if !unloaded {
 			continue
 		}
-		site, err := u.load()
+		site, err := u.load(root)
 		if err != nil {
 			panic(internalError("shipped %s: %v; after a change under data/, regenerate shippedindex.go", join(strings.Join(u.dir, "."), u.name), err))
 		}
@@ -125,14 +116,8 @@ func loadShipped(root *folder, wanted []unloadedCategory) {
 
 // load parses and compiles the category, and moves it from its folder's unloaded map to
 // the folder's children.
-func (u unloadedCategory) load() (categorySite, error) {
+func (u unloadedCategory) load(root *folder) (categorySite, error) {
-	dir := path.Join(append([]string{shippedSource.baseDir}, u.dir...)...)
+	if err := shippedSource.Load(u.dir, u.name, compileInto(root)); err != nil {
-	entries, err := fs.ReadDir(shippedSource.fsys, dir)
-	if err != nil {
-		return categorySite{}, fmt.Errorf("%s: %w", dir, err)
-	}
-	file := u.name + ".json"
-	if err := loadFile(shippedSource, u.in, path.Join(dir, file), file, newCategoryFiles(shippedSource, dir, entries)); err != nil {
 		return categorySite{}, err
 	}
 	delete(u.in.unloaded, u.name)

table.go

@@ -85,12 +85,12 @@ func isTableOption(name string) bool {
 // token arm.
 const inSelector = `[]{}"|`
 
-func compileTable(m map[string]any, segment string, files *categoryFiles) (*table, error) {
+func compileTable(m map[string]any, segment string, readRows func(file string) (string, error)) (*table, error) {
 	o, err := readTableOptions(m)
 	if err != nil {
 		return nil, err
 	}
-	data, err := files.readRows(o.rows)
+	data, err := readRows(o.rows)
 	if err != nil {
 		return nil, err
 	}