From ca40fe0c2c0aceeb6229ce37cf5360ac5a0ff3a4 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Thu, 3 Sep 2026 22:56:56 +0200 Subject: [PATCH 01/21] Add record output to JSON, CSV and SQL --- README.md | 57 ++++++++++++++ cmd/fejkdata/main.go | 94 +++++++++++++++++++++- record.go | 180 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 328 insertions(+), 3 deletions(-) create mode 100644 record.go diff --git a/README.md b/README.md index ab1d65d..594d035 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,8 @@ brace, a bracket or a quote, so the two cannot collide (see | `-s`, `--seed N` | reproducible output | | `-n`, `--repeat N` | render the value N times (up to 1048576), each an independent draw, streamed | | `--separator S` | between repeated values (default a newline) | +| `--format F` | `text` (default), `json`, `csv` or `sql` — a record's columns, one per line | +| `--table T` | the INSERT target for `--format sql` (default: the path's last segment) | | `--list` | print every path, then exit | | `--version`, `-h`, `--help` | print, then exit | @@ -73,6 +75,36 @@ fejkdata --seed 1 --data-path ./mydata sql fejkdata --repeat 100 --data-path ./mydata sql > seed.sql ``` +That `format` string is the free-form spelling — you hand-write the whole row. +For structured output a record writes the row for you. + +### Records + +A record is a template seen as columns: its fields are the columns, its `format` +the whole. `--format json|csv|sql` streams a record per line; the library's +`Record` (below) hands back the columns. Save `mydata/users.json`: + +```json +{ + "format": "{first} {last}", + "first": ["Ada", "Bo"], + "last": ["Lovelace", "Ek"] +} +``` + +```sh +fejkdata --format json users # {"first":"Ada","last":"Lovelace"} +fejkdata --format csv users # first,last → Ada,Lovelace +fejkdata --format sql users # INSERT INTO users (first, last) VALUES ('Ada', 'Lovelace'); +fejkdata --format json --repeat 3 users # three objects, one per line +fejkdata --format sql --table people users # INSERT into another table +``` + +`--repeat` streams that many records — a JSON object per line, a CSV row per +line after a header, an INSERT per line in SQL. Columns render independently, each +a fresh draw, in name order. A column is a string, and each format quotes it as +such; see [Decisions](#decisions) for the typed-scalar and struct-filling scope. + ## Library ```sh @@ -89,6 +121,9 @@ paths := f.List() // every path Fake accepts, sorted v, err = f.FakeTemplate("name: {/sv_SE.person.last}") // compile + render in one call t, err := f.NewTemplate(`{"format":"name: {x}","x":["bosse","lina"]}`) // compile once v = t.Fake() // render many times, no re-parse +r, err := f.Record("users") // one record: each field a column +s := r.JSON() // {"first":"Ada","last":"Lovelace"} +r, err = f.FakeRecord(`{"format":"{x}","x":["a","b"]}`) // compile + render inline ``` | Option | | @@ -98,6 +133,11 @@ v = t.Fake() // render many times, | `WithDataFS(fsys)` | layer an `fs.FS`, such as your own `embed.FS` | | `WithoutShippedData()` | load only what you give | +A `*Record` carries its columns via `Fields()`, and serializes them with `JSON()` +(one object), `CSVHeader()`/`CSVLine()`, or `SQLInsert(table)` — the same three +shapes the CLI's `--format` streams. `Record` and `FakeRecord` take a record; a +path or template that is not one — a bare string, a choice, or a folder — errors. + A `*Generator` is safe for concurrent use; a seeded sequence is reproducible only when drawn from one goroutine. Changing how a value is composed shifts the seeded stream for that value and everything drawn after it. @@ -454,6 +494,22 @@ tokens add cost in proportion to the output. - **Samples say what they emit, transforms what they do.** `{upper(2)}` is two letters, `{uppercase(x)}` is `x` upper-cased; one name for both would turn on whether the argument looks like a number. +- **A record is a template seen as columns, not a second schema format.** A + template's `format` composes its fields into one string; `Record` and + `--format` project the same fields as columns. Two views of one dataset, so a + record author writes the same JSON they already know, and a column is the same + field `Fake` renders by dotted path. +- **Records emit text; typed scalars are out.** Every value fejkdata yields is a + string, so JSON, CSV and SQL each quote a column as text (`"42"`, `'42'`) rather + than guess a number or a boolean. Emitting unquoted numbers or booleans would + need a per-column `kind`, a parallel scalar system in a format whose promise is + "text means what it says". A column's check digit, number or id is still + valid-by-construction through a builtin; it is serialized as text. +- **Filling a Go struct is out of scope.** `Record.Fields()` returns the columns a + caller maps onto a struct themselves. gofakeit's `fake:"{firstname}"` tags + reflect over an arbitrary struct type and cast into its fields — a different + concern from "data lives in JSON", and one a JSON record feeds without fejkdata + owning the reflection. - **The performance gate asserts allocations, not wall-clock time.** `AllocsPerRun` is deterministic across machines, so a ±10% ceiling does not flake under CI load, while time varies with the machine and its neighbours. A rendering slowdown @@ -500,6 +556,7 @@ fejkdata.go Generator, New, options, the embedded data set, List node.go the node model and JSON -> node compilation path.go the dotted-path walk, and proving a path resolves render.go Fake and the recursive renderer (choices, format strings, expansions) +record.go records: Record, the JSON/CSV/SQL serializers, and their entry points inline.go inline templates: Template, NewTemplate, FakeTemplate, and their compile and link template.go the {token} grammar: scanning, tokens, operands, validation, compiling a format hold.go the hold: one draw per expansion for paths and operands, and its fences diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index bf69ed0..3dbd5fb 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -34,13 +34,19 @@ or a quote). Templates reach the data by reference from the root — argument carrying a bracket, a closing brace or a quote but no valid JSON names neither. +With --format json, csv or sql the argument must name a record — a template whose +fields are its columns — and each render is streamed as one JSON object, one CSV +row (after a header), or one INSERT. + -d, --data-path D a data directory to layer over the shipped data (repeatable; last wins on a clash) + --format F output form: text (default), json, csv or sql -h, --help print this help, then exit --list list the paths the data offers, then exit --no-shipped-data load only the --data-path directories -n, --repeat N render the value N times, 1..1048576 (default 1) -s, --seed N seed for reproducible output --separator S string between repeated values (default newline) + --table T the INSERT target for --format sql (default: the path's last segment) --version print the version, then exit Flags may come before or after ; -- ends the flags. A short flag's @@ -49,6 +55,8 @@ value attaches or follows (-n3, -n 3); short flags bundle (-hn 3). type invocation struct { dirs []string + format string + formatSet bool help bool list bool noShipped bool @@ -59,6 +67,8 @@ type invocation struct { seeded bool separator string separatorSet bool + table string + tableSet bool version bool } @@ -71,6 +81,7 @@ type flagDef struct { var flagDefs = []flagDef{ {"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }}, + {"format", "", true, func(in *invocation, v string) error { in.format, in.formatSet = v, true; return nil }}, {"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }}, {"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }}, {"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }}, @@ -91,6 +102,7 @@ var flagDefs = []flagDef{ return nil }}, {"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }}, + {"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }}, {"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }}, } @@ -158,7 +170,7 @@ func splitFlags(arg string) ([]flagArg, error) { } func parseArgs(argv []string) (invocation, error) { - in := invocation{repeat: 1, separator: "\n"} + in := invocation{repeat: 1, separator: "\n", format: "text"} for i := 0; i < len(argv); i++ { arg := argv[i] if arg == "--" { @@ -196,14 +208,41 @@ func parseArgs(argv []string) (invocation, error) { return in, nil } +// recordFormat reports whether the format names a record output (json, csv or +// sql) rather than the default text. +func (in invocation) recordFormat() bool { + return in.formatSet && in.format != "text" +} + +// checkFlags rejects a flag value or combination that cannot run. +func (in invocation) checkFlags() error { + if in.formatSet { + switch in.format { + case "text", "json", "csv", "sql": + default: + return fmt.Errorf("--format takes text, json, csv or sql, got %q", in.format) + } + } + if in.tableSet && in.format != "sql" { + return errors.New("--table names the INSERT target, so it needs --format sql") + } + if in.recordFormat() && in.separatorSet { + return errors.New("--separator joins text values, so it has no effect with --format " + in.format) + } + return nil +} + // check rejects a flag combination or an argument that cannot run, and reports // what the argument names, so its shape is settled before any data is read. func (in invocation) check() (argKind, error) { + if err := in.checkFlags(); err != nil { + return argPath, err + } if in.list && len(in.paths) > 0 { return argPath, errors.New("--list takes no path") } - if in.list && (in.repeatSet || in.separatorSet) { - return argPath, errors.New("--list takes no --repeat or --separator") + if in.list && (in.repeatSet || in.separatorSet || in.formatSet || in.tableSet) { + return argPath, errors.New("--list takes no --repeat, --separator, --format or --table") } if in.list { return argPath, nil @@ -234,6 +273,9 @@ func (in invocation) options() []fejkdata.Option { // from Flush, bufio keeping the first one. func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) error { arg := in.paths[0] + if in.recordFormat() { + return in.writeRecords(f, kind, arg, w) + } draw := func() (string, error) { return f.Fake(arg) } if kind == argTemplate { t, err := f.NewTemplate(arg) @@ -257,6 +299,52 @@ func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) err return out.Flush() } +// writeRecords streams a record per line in the chosen format: one JSON object +// per line, a CSV header then a row per line, or one INSERT per line. +func (in invocation) writeRecords(f *fejkdata.Generator, kind argKind, arg string, w io.Writer) error { + draw := func() (*fejkdata.Record, error) { return f.Record(arg) } + if kind == argTemplate { + t, err := f.NewRecordTemplate(arg) + if err != nil { + return templateError{err} + } + draw = func() (*fejkdata.Record, error) { return t.Fake(), nil } + } + out := bufio.NewWriter(w) + table := in.table + if table == "" { + table = defaultTable(arg, kind) + } + for i := 0; i < in.repeat; i++ { + r, err := draw() + if err != nil { + return err + } + switch in.format { + case "json": + fmt.Fprintln(out, r.JSON()) + case "csv": + if i == 0 { + fmt.Fprintln(out, r.CSVHeader()) + } + fmt.Fprintln(out, r.CSVLine()) + case "sql": + fmt.Fprintln(out, r.SQLInsert(table)) + } + } + return out.Flush() +} + +// defaultTable names the INSERT target when --table is absent: the path's last +// segment, or "records" for an inline template that sits in no folder. +func defaultTable(arg string, kind argKind) string { + if kind == argTemplate { + return "records" + } + segments := strings.Split(arg, ".") + return segments[len(segments)-1] +} + // templateError marks a render failure that is the argument's own fault — an // inline template that does not compile. run reports it as misuse (exit 2, with a // pointer to --help), unlike an unknown path, which is a runtime error (exit 1). diff --git a/record.go b/record.go new file mode 100644 index 0000000..9ef7cbd --- /dev/null +++ b/record.go @@ -0,0 +1,180 @@ +package fejkdata + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "sort" + "strings" +) + +// Field is one rendered column of a record. +type Field struct { + Name string + Value string +} + +// Record is one record rendered from a template: every direct field is a column, +// drawn independently and listed in name order. The template's format is the +// string a [Generator.Fake] call renders; a record is its inverse — each field +// projected as a column instead of composed. +type Record struct { + fields []Field +} + +// Fields returns the record's columns in name order. +func (r *Record) Fields() []Field { + return append([]Field(nil), r.fields...) +} + +// JSON renders the record as one JSON object, every column a string. +func (r *Record) JSON() string { + m := make(map[string]string, len(r.fields)) + for _, f := range r.fields { + m[f.Name] = f.Value + } + b, _ := json.Marshal(m) + return string(b) +} + +// CSVHeader renders the column names as one CSV header line. +func (r *Record) CSVHeader() string { + return csvLine(r.names()) +} + +// CSVLine renders the column values as one CSV row. +func (r *Record) CSVLine() string { + return csvLine(r.values()) +} + +func (r *Record) names() []string { + out := make([]string, len(r.fields)) + for i, f := range r.fields { + out[i] = f.Name + } + return out +} + +func (r *Record) values() []string { + out := make([]string, len(r.fields)) + for i, f := range r.fields { + out[i] = f.Value + } + return out +} + +func csvLine(cols []string) string { + var b strings.Builder + w := csv.NewWriter(&b) + _ = w.Write(cols) + w.Flush() + return strings.TrimSuffix(b.String(), "\n") +} + +// SQLInsert renders the record as one INSERT statement into table, every column a +// single-quoted string literal. +func (r *Record) SQLInsert(table string) string { + cols := make([]string, len(r.fields)) + vals := make([]string, len(r.fields)) + for i, f := range r.fields { + cols[i] = f.Name + vals[i] = "'" + strings.ReplaceAll(f.Value, "'", "''") + "'" + } + return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);", table, strings.Join(cols, ", "), strings.Join(vals, ", ")) +} + +// Record renders a path as one record: the template it names, with each direct +// field drawn as a column. A path naming a folder or a value that is not a +// template — a bare string or a choice, which have no fields — is an error. +func (f *Generator) Record(path string) (*Record, error) { + f.mu.Lock() + defer f.mu.Unlock() + n, err := descend(f.rand, &group{children: f.categories}, strings.Split(path, ".")) + if err != nil { + return nil, fmt.Errorf("fejkdata: %s: %w", path, err) + } + if _, ok := n.(*group); ok { + return nil, fmt.Errorf("fejkdata: %s names a folder, not a value", path) + } + t, ok := n.(*template) + if !ok { + return nil, fmt.Errorf("fejkdata: %s does not name a record; it has no fields to project", path) + } + r := renderRecord(f.rand, t) + if len(r.fields) == 0 { + return nil, fmt.Errorf("fejkdata: %s has no fields, so no columns", path) + } + return r, nil +} + +// RecordTemplate is an inline record compiled, referenced and validated once, +// ready to render many times with [RecordTemplate.Fake]. +type RecordTemplate struct { + g *Generator + n node +} + +// Fake renders the record with one draw. +func (t *RecordTemplate) Fake() *Record { + t.g.mu.Lock() + defer t.g.mu.Unlock() + return renderRecord(t.g.rand, t.n.(*template)) +} + +// NewRecordTemplate compiles an inline record — a JSON object with a format and +// fields — and binds its references against the loaded tree. +func (f *Generator) NewRecordTemplate(input string) (*RecordTemplate, error) { + n, err := compileInput(input) + if err != nil { + return nil, fmt.Errorf("fejkdata: %w", err) + } + t, ok := n.(*template) + if !ok { + return nil, fmt.Errorf("fejkdata: an inline record is a JSON object with a format and fields, not a choice") + } + if len(recordColumns(t)) == 0 { + return nil, fmt.Errorf("fejkdata: an inline record needs at least one field to project as a column") + } + scope := inlineScope(t) + if err := linkNodeRefs(scope, f.categories); err != nil { + return nil, fmt.Errorf("fejkdata: %w", err) + } + if err := checkScope(scope); err != nil { + return nil, fmt.Errorf("fejkdata: %w", err) + } + return &RecordTemplate{g: f, n: t}, nil +} + +// FakeRecord compiles and renders an inline record in one call. +func (f *Generator) FakeRecord(input string) (*Record, error) { + t, err := f.NewRecordTemplate(input) + if err != nil { + return nil, err + } + return t.Fake(), nil +} + +// renderRecord projects a template's direct fields as columns, drawn once each, +// in name order. A {/path} binding is a render edge, not a column, so it is +// skipped the same way List and the graph do. +func renderRecord(s *session, t *template) *Record { + r := &Record{} + for _, name := range recordColumns(t) { + r.fields = append(r.fields, Field{Name: name, Value: render(s, t.fields[name])}) + } + return r +} + +// recordColumns is the sorted non-reference field names — the columns a record +// projects. A {/path} binding is carried in fields under its root path, so only a +// name that is not a reference is a column. +func recordColumns(t *template) []string { + var names []string + for name := range t.fields { + if !isRef(name) { + names = append(names, name) + } + } + sort.Strings(names) + return names +} -- 2.52.0 From 7383e8d0ca68e86a94826c562ade3e6ee1faa121 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Thu, 3 Sep 2026 22:56:56 +0200 Subject: [PATCH 02/21] Test record output and its serializers --- cmd/fejkdata/main_test.go | 104 ++++++++++++++++++++++ readme_test.go | 25 ++++++ record_test.go | 176 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 305 insertions(+) create mode 100644 record_test.go diff --git a/cmd/fejkdata/main_test.go b/cmd/fejkdata/main_test.go index 258f740..9870b1a 100644 --- a/cmd/fejkdata/main_test.go +++ b/cmd/fejkdata/main_test.go @@ -2,6 +2,8 @@ package main import ( "bytes" + "encoding/csv" + "encoding/json" "os" "path/filepath" "regexp" @@ -450,6 +452,108 @@ func TestRunEmptyDataPathIsNamed(t *testing.T) { } } +func recordDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + content := `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"]}` + if err := os.WriteFile(filepath.Join(dir, "users.json"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return dir +} + +func TestRunRecordJSON(t *testing.T) { + code, out, errb := runOut("--seed", "1", "--format", "json", "--data-path", recordDir(t), "users") + if code != 0 { + t.Fatalf("run = %d, stderr=%q", code, errb) + } + var m map[string]string + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &m); err != nil { + t.Fatalf("json output is not one valid JSON object: %v\n%q", err, out) + } + if len(m) != 2 || m["first"] == "" || m["last"] == "" { + t.Fatalf("json output = %q, want first and last columns", out) + } +} + +func TestRunRecordCSVRoundTrips(t *testing.T) { + code, out, errb := runOut("--seed", "1", "--format", "csv", "--repeat", "3", "--data-path", recordDir(t), "users") + if code != 0 { + t.Fatalf("run = %d, stderr=%q", code, errb) + } + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + if len(lines) != 4 { + t.Fatalf("csv output = %d lines %q, want a header and 3 rows", len(lines), out) + } + if lines[0] != "first,last" { + t.Fatalf("csv header = %q, want first,last", lines[0]) + } + r, err := csv.NewReader(strings.NewReader(out)).ReadAll() + if err != nil { + t.Fatalf("csv output is not valid CSV: %v", err) + } + if len(r) != 4 || len(r[0]) != 2 { + t.Fatalf("csv parsed to %v, want 4 records of 2 fields", r) + } +} + +func TestRunRecordSQL(t *testing.T) { + code, out, errb := runOut("--seed", "1", "--format", "sql", "--repeat", "2", "--table", "people", "--data-path", recordDir(t), "users") + if code != 0 { + t.Fatalf("run = %d, stderr=%q", code, errb) + } + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + if len(lines) != 2 || !strings.HasPrefix(lines[0], "INSERT INTO people (first, last) VALUES (") { + t.Fatalf("sql output = %q, want two INSERT INTO people statements", out) + } +} + +func TestRunRecordDefaultTable(t *testing.T) { + code, out, errb := runOut("--seed", "1", "--format", "sql", "--data-path", recordDir(t), "users") + if code != 0 || !strings.HasPrefix(out, "INSERT INTO users (") { + t.Fatalf("default table = %d, %q, want the path's last segment; stderr %q", code, out, errb) + } +} + +func TestRunRecordInlineTemplate(t *testing.T) { + code, out, errb := runOut("--seed", "1", "--format", "json", `{"format":"{x}","x":["a","b"]}`) + if code != 0 { + t.Fatalf("inline record = %d, stderr=%q", code, errb) + } + var m map[string]string + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &m); err != nil || m["x"] != "a" && m["x"] != "b" { + t.Fatalf("inline record json = %q, want a column x (err %v)", out, err) + } +} + +func TestRunRecordMisuse(t *testing.T) { + for _, c := range []struct { + args []string + want string + }{ + {[]string{"--format", "yaml", "users"}, "--format takes text, json, csv or sql"}, + {[]string{"--format", "json", "--separator", ",", "users"}, "--separator joins text values"}, + {[]string{"--table", "t", "users"}, "--table names the INSERT target"}, + {[]string{"--format", "json", "--table", "t", "users"}, "--table names the INSERT target"}, + } { + code, out, errb := runOut(c.args...) + if code != 2 || out != "" || !strings.Contains(errb, c.want) { + t.Errorf("run(%v) = %d, %q, %q; want misuse naming %q", c.args, code, out, errb, c.want) + } + } +} + +func TestRunRecordOnABareValueIsRuntimeError(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "word.json"), []byte(`["a","b"]`), 0o644); err != nil { + t.Fatal(err) + } + code, _, errb := runOut("--format", "json", "--data-path", dir, "word") + if code != 1 { + t.Fatalf("record on a choice = %d, want exit 1 (runtime error), stderr %q", code, errb) + } +} + func TestRunNumbersFollowTheShell(t *testing.T) { _, want, _ := runOut("--seed", "7", "--repeat", "3", "sv_SE.word") code, got, errb := runOut("--seed", "007", "--repeat", "+3", "sv_SE.word") diff --git a/readme_test.go b/readme_test.go index 594fabc..38ff2ee 100644 --- a/readme_test.go +++ b/readme_test.go @@ -43,6 +43,31 @@ func TestReadmeExamplesLoadAndRender(t *testing.T) { } } +func TestReadmeRecordExample(t *testing.T) { + src := readme(t) + section := src[strings.Index(src, "### Records"):] + section = section[:strings.Index(section, "## Library")] + block := jsonBlock.FindStringSubmatch(section) + if block == nil { + t.Fatal("README lost the Records example") + } + f, err := New(WithDataPath(writeData(t, map[string]string{"users": block[1]})), WithSeed(1)) + if err != nil { + t.Fatal(err) + } + r, err := f.Record("users") + if err != nil { + t.Fatal(err) + } + got := r.Fields() + if len(got) != 2 || got[0].Name != "first" || got[1].Name != "last" { + t.Fatalf("record columns = %v, want first, last", got) + } + if r.CSVHeader() != "first,last" || !strings.HasPrefix(r.SQLInsert("users"), "INSERT INTO users (first, last) VALUES (") { + t.Fatalf("serializers = %q, %q, want first,last and an INSERT", r.CSVHeader(), r.SQLInsert("users")) + } +} + func TestReadmeSQLExampleOutput(t *testing.T) { src := readme(t) src = src[strings.Index(src, "### Your own data"):] diff --git a/record_test.go b/record_test.go new file mode 100644 index 0000000..d330938 --- /dev/null +++ b/record_test.go @@ -0,0 +1,176 @@ +package fejkdata + +import ( + "encoding/csv" + "encoding/json" + "strings" + "testing" +) + +func recordCat(t *testing.T) *Generator { + t.Helper() + dir := writeData(t, map[string]string{ + "users": `{ + "format": "{first} {last}", + "first": ["Ada", "Bo"], + "last": ["Lovelace", "Ek"] + }`, + }) + return newGenerator(t, dir, WithSeed(1)) +} + +func TestRecordProjectsFieldsAsColumns(t *testing.T) { + f := recordCat(t) + r, err := f.Record("users") + if err != nil { + t.Fatal(err) + } + got := r.Fields() + if len(got) != 2 || got[0].Name != "first" || got[1].Name != "last" { + t.Fatalf("Record.Fields() = %v, want columns first, last in name order", got) + } + if (got[0].Value != "Ada" && got[0].Value != "Bo") || (got[1].Value != "Lovelace" && got[1].Value != "Ek") { + t.Fatalf("columns = %v, want the field values", got) + } +} + +func TestRecordIsDeterministic(t *testing.T) { + a, b := recordCat(t), recordCat(t) + for i := 0; i < 20; i++ { + x, _ := a.Record("users") + y, _ := b.Record("users") + if x.JSON() != y.JSON() { + t.Fatalf("same seed diverged: %s != %s", x.JSON(), y.JSON()) + } + } +} + +func TestRecordSkipsReferenceBindings(t *testing.T) { + dir := writeData(t, map[string]string{ + "name": `{"format": "{first} {last}", "first": "Ada", "last": "Lovelace"}`, + "user": `{"format": "they are {/name}", "id": "1"}`, + }) + f := newGenerator(t, dir, WithSeed(1)) + r, err := f.Record("user") + if err != nil { + t.Fatal(err) + } + got := r.Fields() + if len(got) != 1 || got[0].Name != "id" { + t.Fatalf("Record.Fields() = %v, want only the id column (a {/path} is not a column)", got) + } +} + +func TestRecordErrors(t *testing.T) { + dir := writeData(t, map[string]string{ + "bare": `"hello"`, + "pick": `["a", "b"]`, + "group/cat": `"x"`, + }) + f := newGenerator(t, dir, WithSeed(1)) + for _, c := range []struct { + path string + want string + }{ + {"bare", "has no fields, so no columns"}, + {"pick", "does not name a record"}, + {"group", "names a folder"}, + {"nope", "no entry"}, + } { + if _, err := f.Record(c.path); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("Record(%q) = %v, want an error containing %q", c.path, err, c.want) + } + } +} + +func TestRecordJSON(t *testing.T) { + f := recordCat(t) + r, err := f.Record("users") + if err != nil { + t.Fatal(err) + } + var m map[string]string + if err := json.Unmarshal([]byte(r.JSON()), &m); err != nil { + t.Fatalf("Record.JSON() is not valid JSON: %v\n%s", err, r.JSON()) + } + if len(m) != 2 || (m["first"] != "Ada" && m["first"] != "Bo") { + t.Fatalf("Record.JSON() = %s, want two addressable columns", r.JSON()) + } + for _, f := range r.Fields() { + if m[f.Name] != f.Value { + t.Fatalf("JSON column %q = %q, want %q", f.Name, m[f.Name], f.Value) + } + } +} + +func TestRecordCSV(t *testing.T) { + dir := writeData(t, map[string]string{ + "note": `{"format": "{word}, {word}", "word": ["a", "b,c", "d\"e", "f\n"]}`, + }) + f := newGenerator(t, dir, WithSeed(1)) + r, err := f.Record("note") + if err != nil { + t.Fatal(err) + } + if got := r.CSVHeader(); got != "word" { + t.Errorf("CSVHeader() = %q, want word", got) + } + rec, err := csv.NewReader(strings.NewReader(r.CSVLine() + "\n")).Read() + if err != nil { + t.Fatalf("CSVLine() is not valid CSV: %v\n%q", err, r.CSVLine()) + } + if len(rec) != 1 || rec[0] != r.Fields()[0].Value { + t.Fatalf("CSVLine() = %v, want the field value round-tripped", rec) + } +} + +func TestRecordSQLInsert(t *testing.T) { + dir := writeData(t, map[string]string{ + "person": `{"format": "{last}", "last": "O'Brien"}`, + }) + f := newGenerator(t, dir, WithSeed(1)) + r, err := f.Record("person") + if err != nil { + t.Fatal(err) + } + got := r.SQLInsert("people") + if got != "INSERT INTO people (last) VALUES ('O''Brien');" { + t.Fatalf("SQLInsert() = %q, want the single quote doubled", got) + } +} + +func TestFakeRecordAndTemplate(t *testing.T) { + f := recordCat(t) + in := `{"format":"{x} {y}","x":["1","2"],"y":["3","4"]}` + want, err := f.FakeRecord(in) + if err != nil { + t.Fatalf("FakeRecord: %v", err) + } + if len(want.Fields()) != 2 { + t.Fatalf("FakeRecord fields = %v, want two columns", want.Fields()) + } + reusable, err := f.NewRecordTemplate(in) + if err != nil { + t.Fatalf("NewRecordTemplate: %v", err) + } + for i := 0; i < 20; i++ { + if got := reusable.Fake(); len(got.Fields()) != 2 { + t.Fatalf("RecordTemplate.Fake() = %v, want two columns", got.Fields()) + } + } +} + +func TestInlineRecordErrors(t *testing.T) { + f := recordCat(t) + for _, c := range []struct { + input string + want string + }{ + {`"hello"`, "needs at least one field"}, + {`["a","b"]`, "a choice"}, + } { + if _, err := f.FakeRecord(c.input); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("FakeRecord(%q) = %v, want an error naming %q", c.input, err, c.want) + } + } +} -- 2.52.0 From 6a8aa96b20c8344c5b76f4def7829532e728912b Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Thu, 3 Sep 2026 23:12:45 +0200 Subject: [PATCH 03/21] Quote SQL identifiers and share reference draws across record columns --- README.md | 35 +++++++++++++++++++++------ cmd/fejkdata/main_test.go | 4 +-- hold.go | 20 +++++++++------ hold_test.go | 2 +- readme_test.go | 2 +- record.go | 20 +++++++++------ record_test.go | 51 ++++++++++++++++++++++++++++++++++++--- render.go | 22 +++++++++++------ template_test.go | 2 +- 9 files changed, 121 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 594d035..835227b 100644 --- a/README.md +++ b/README.md @@ -93,17 +93,28 @@ the whole. `--format json|csv|sql` streams a record per line; the library's ``` ```sh -fejkdata --format json users # {"first":"Ada","last":"Lovelace"} -fejkdata --format csv users # first,last → Ada,Lovelace -fejkdata --format sql users # INSERT INTO users (first, last) VALUES ('Ada', 'Lovelace'); -fejkdata --format json --repeat 3 users # three objects, one per line +fejkdata --format json users # {"first":"Ada","last":"Lovelace"} +fejkdata --format csv users # first,last → Ada,Lovelace +fejkdata --format sql users # INSERT INTO "users" ("first", "last") VALUES ('Ada', 'Lovelace'); +fejkdata --format json --repeat 3 users # three objects, one per line fejkdata --format sql --table people users # INSERT into another table ``` `--repeat` streams that many records — a JSON object per line, a CSV row per -line after a header, an INSERT per line in SQL. Columns render independently, each -a fresh draw, in name order. A column is a string, and each format quotes it as -such; see [Decisions](#decisions) for the typed-scalar and struct-filling scope. +line after a header, an INSERT per line in SQL. Only a category-level template is +a record; a field, choice or folder errors. Column identifiers are double-quoted +in SQL, so a hyphenated field like `postal-code` stays valid. A column is a +string, and each format quotes it as such; see [Decisions](#decisions) for the +typed-scalar, correlation and struct-filling scope. + +A record written only to emit columns still needs a `format` — the grammar's one +required key — so `"format": ""` carries the fields with an inert format: it +renders nothing by `Fake`, and is compiled only so the tree's fences still run. +The columns are the point, and their facts stay together: two columns that +reference one category — `{/currency.code}` and `{/currency.symbol}` — share one +draw of it, so the record is internally consistent. A field hold, transform or +operand ties fields together within one column as always (see +[Correlated fields](#correlated-fields) and [Decisions](#decisions)). ## Library @@ -498,13 +509,21 @@ tokens add cost in proportion to the output. template's `format` composes its fields into one string; `Record` and `--format` project the same fields as columns. Two views of one dataset, so a record author writes the same JSON they already know, and a column is the same - field `Fake` renders by dotted path. + field `Fake` renders by dotted path. The `format` is inert to a record — a + record-only template writes `"format": ""` — but it is compiled and fenced, so + a template that loads renders as whichever shape is asked for. - **Records emit text; typed scalars are out.** Every value fejkdata yields is a string, so JSON, CSV and SQL each quote a column as text (`"42"`, `'42'`) rather than guess a number or a boolean. Emitting unquoted numbers or booleans would need a per-column `kind`, a parallel scalar system in a format whose promise is "text means what it says". A column's check digit, number or id is still valid-by-construction through a builtin; it is serialized as text. +- **A record's columns share one reference draw.** Two columns that reference one + category — `{/currency.code}` beside `{/currency.symbol}` — read one draw of it, + so a record's facts agree the way a template's [correlated + fields](#correlated-fields) do. Only references share across columns: a sibling + field is local to its own column, so `first` does not silently bind to a + `first` in the column next to it. - **Filling a Go struct is out of scope.** `Record.Fields()` returns the columns a caller maps onto a struct themselves. gofakeit's `fake:"{firstname}"` tags reflect over an arbitrary struct type and cast into its fields — a different diff --git a/cmd/fejkdata/main_test.go b/cmd/fejkdata/main_test.go index 9870b1a..d56625a 100644 --- a/cmd/fejkdata/main_test.go +++ b/cmd/fejkdata/main_test.go @@ -503,14 +503,14 @@ func TestRunRecordSQL(t *testing.T) { t.Fatalf("run = %d, stderr=%q", code, errb) } lines := strings.Split(strings.TrimRight(out, "\n"), "\n") - if len(lines) != 2 || !strings.HasPrefix(lines[0], "INSERT INTO people (first, last) VALUES (") { + if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) { t.Fatalf("sql output = %q, want two INSERT INTO people statements", out) } } func TestRunRecordDefaultTable(t *testing.T) { code, out, errb := runOut("--seed", "1", "--format", "sql", "--data-path", recordDir(t), "users") - if code != 0 || !strings.HasPrefix(out, "INSERT INTO users (") { + if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) { t.Fatalf("default table = %d, %q, want the path's last segment; stderr %q", code, out, errb) } } diff --git a/hold.go b/hold.go index 908939f..a1eddb3 100644 --- a/hold.go +++ b/hold.go @@ -267,14 +267,20 @@ type draws struct { // gives one value, and a shown operand is the operand computed. Every other name is // drawn afresh, so {word} {word} still draws twice. checkTokens, checkPath and // linkRefs prove every step, so the walk cannot fail. -func readField(s *session, t *template, held *draws, a arm) string { +func readField(s *session, t *template, held, shared *draws, a arm) string { if !t.held[a.key] { if len(a.tail) > 0 { panic(fmt.Sprintf("fejkdata: %q reads a path into %q, which the expansion does not hold", a.name, a.key)) } - return render(s, t.fields[a.key]) + return renderShared(s, t.fields[a.key], shared) } - if v, read := held.value[a.path]; read { + // A reference names a shared source, so a record shares its draw across the + // columns; a sibling field is drawn per expansion as always. + d := held + if isRef(a.key) && shared != nil { + d = shared + } + if v, read := d.value[a.path]; read { return v } var v string @@ -286,16 +292,16 @@ func readField(s *session, t *template, held *draws, a arm) string { if consumed := len(a.tail) - len(rest); consumed > 0 { key = a.steps[consumed-1] } - n, drew := held.variant[key] + n, drew := d.variant[key] if !drew { n = drawn(s, c) - held.variant[key] = n + d.variant[key] = n } return []node{n}, nil }, - leaf: func(n node) error { v = render(s, n); return nil }, + leaf: func(n node) error { v = renderShared(s, n, shared); return nil }, }) - held.value[a.path] = v + d.value[a.path] = v return v } diff --git a/hold_test.go b/hold_test.go index 3827d55..ac1ac56 100644 --- a/hold_test.go +++ b/hold_test.go @@ -718,6 +718,6 @@ func TestReadFieldPanicsOnAnUnheldPath(t *testing.T) { t.Fatal("not a template") } mustPanic(t, "unheld arm with a path", func() { - readField(engine(1).rand, tm, nil, arm{name: "w.x", key: "w", tail: []string{"x"}, path: "w.x"}) + readField(engine(1).rand, tm, nil, nil, arm{name: "w.x", key: "w", tail: []string{"x"}, path: "w.x"}) }) } diff --git a/readme_test.go b/readme_test.go index 38ff2ee..0a12f81 100644 --- a/readme_test.go +++ b/readme_test.go @@ -63,7 +63,7 @@ func TestReadmeRecordExample(t *testing.T) { if len(got) != 2 || got[0].Name != "first" || got[1].Name != "last" { t.Fatalf("record columns = %v, want first, last", got) } - if r.CSVHeader() != "first,last" || !strings.HasPrefix(r.SQLInsert("users"), "INSERT INTO users (first, last) VALUES (") { + if r.CSVHeader() != "first,last" || !strings.HasPrefix(r.SQLInsert("users"), `INSERT INTO "users" ("first", "last") VALUES (`) { t.Fatalf("serializers = %q, %q, want first,last and an INSERT", r.CSVHeader(), r.SQLInsert("users")) } } diff --git a/record.go b/record.go index 9ef7cbd..0b7d1d3 100644 --- a/record.go +++ b/record.go @@ -71,16 +71,20 @@ func csvLine(cols []string) string { return strings.TrimSuffix(b.String(), "\n") } -// SQLInsert renders the record as one INSERT statement into table, every column a -// single-quoted string literal. +// SQLInsert renders the record as one INSERT statement into table: identifiers in +// ANSI double quotes, every value a single-quoted string literal. func (r *Record) SQLInsert(table string) string { cols := make([]string, len(r.fields)) vals := make([]string, len(r.fields)) for i, f := range r.fields { - cols[i] = f.Name + cols[i] = quoteIdent(f.Name) vals[i] = "'" + strings.ReplaceAll(f.Value, "'", "''") + "'" } - return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);", table, strings.Join(cols, ", "), strings.Join(vals, ", ")) + return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);", quoteIdent(table), strings.Join(cols, ", "), strings.Join(vals, ", ")) +} + +func quoteIdent(s string) string { + return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` } // Record renders a path as one record: the template it names, with each direct @@ -98,7 +102,7 @@ func (f *Generator) Record(path string) (*Record, error) { } t, ok := n.(*template) if !ok { - return nil, fmt.Errorf("fejkdata: %s does not name a record; it has no fields to project", path) + return nil, fmt.Errorf("fejkdata: %s names a choice, not a template; only a category-level template is a record", path) } r := renderRecord(f.rand, t) if len(r.fields) == 0 { @@ -156,11 +160,13 @@ func (f *Generator) FakeRecord(input string) (*Record, error) { // renderRecord projects a template's direct fields as columns, drawn once each, // in name order. A {/path} binding is a render edge, not a column, so it is -// skipped the same way List and the graph do. +// skipped the same way List and the graph do. The columns share one draw context, +// so two columns that reference one category read one draw of it. func renderRecord(s *session, t *template) *Record { + shared := &draws{variant: map[string]node{}, value: map[string]string{}} r := &Record{} for _, name := range recordColumns(t) { - r.fields = append(r.fields, Field{Name: name, Value: render(s, t.fields[name])}) + r.fields = append(r.fields, Field{Name: name, Value: renderShared(s, t.fields[name], shared)}) } return r } diff --git a/record_test.go b/record_test.go index d330938..7493c88 100644 --- a/record_test.go +++ b/record_test.go @@ -73,7 +73,7 @@ func TestRecordErrors(t *testing.T) { want string }{ {"bare", "has no fields, so no columns"}, - {"pick", "does not name a record"}, + {"pick", "names a choice"}, {"group", "names a folder"}, {"nope", "no entry"}, } { @@ -134,8 +134,53 @@ func TestRecordSQLInsert(t *testing.T) { t.Fatal(err) } got := r.SQLInsert("people") - if got != "INSERT INTO people (last) VALUES ('O''Brien');" { - t.Fatalf("SQLInsert() = %q, want the single quote doubled", got) + if got != `INSERT INTO "people" ("last") VALUES ('O''Brien');` { + t.Fatalf("SQLInsert() = %q, want quoted identifiers and the single quote doubled", got) + } +} + +func TestRecordSharesAReferenceAcrossColumns(t *testing.T) { + dir := writeData(t, map[string]string{ + "currency": `[{"format":"{code}","code":"AUD","symbol":"$"},{"format":"{code}","code":"EUR","symbol":"€"}]`, + "price": `{"format":"","code":"{/currency.code}","symbol":"{/currency.symbol}"}`, + }) + f := newGenerator(t, dir, WithSeed(1)) + for i := 0; i < 100; i++ { + r, err := f.Record("price") + if err != nil { + t.Fatal(err) + } + m := map[string]string{} + for _, c := range r.Fields() { + m[c.Name] = c.Value + } + switch m["code"] { + case "AUD": + if m["symbol"] != "$" { + t.Fatalf("record %q: code AUD but symbol %q, want one currency draw across columns", r.JSON(), m["symbol"]) + } + case "EUR": + if m["symbol"] != "€" { + t.Fatalf("record %q: code EUR but symbol %q, want one currency draw across columns", r.JSON(), m["symbol"]) + } + default: + t.Fatalf("record %q has unexpected code %q", r.JSON(), m["code"]) + } + } +} + +func TestRecordSQLQuotesIdentifiers(t *testing.T) { + dir := writeData(t, map[string]string{ + "row": `{"format": "", "postal-code": "1", "street-number": "2"}`, + }) + f := newGenerator(t, dir, WithSeed(1)) + r, err := f.Record("row") + if err != nil { + t.Fatal(err) + } + got := r.SQLInsert("my-table") + if got != `INSERT INTO "my-table" ("postal-code", "street-number") VALUES ('1', '2');` { + t.Fatalf("SQLInsert() = %q, want hyphenated identifiers and table quoted", got) } } diff --git a/render.go b/render.go index 2d998fb..b930cac 100644 --- a/render.go +++ b/render.go @@ -52,15 +52,22 @@ func descend(s *session, root node, segments []string) (node, error) { // render evaluates a compiled node to a string. compile validates every node up // front, so rendering a compiled tree cannot fail. func render(s *session, n node) string { + return renderShared(s, n, nil) +} + +// renderShared is render with a shared draw context: the draws a record shares +// across its columns. A nil shared means a standalone render, where a reference +// is drawn per expansion as it always has been. +func renderShared(s *session, n node, shared *draws) string { switch n := n.(type) { case *choice: - return render(s, pick(s, n)) + return renderShared(s, pick(s, n), shared) case *template: if n.repeat == 1 { if n.fixed { return n.lit } - return expand(s, n) + return expand(s, n, shared) } var b strings.Builder b.Grow(n.repeat * (n.grow + len(n.separator))) @@ -68,7 +75,7 @@ func render(s *session, n node) string { if i > 0 { b.WriteString(n.separator) } - b.WriteString(expand(s, n)) + b.WriteString(expand(s, n, shared)) } return b.String() default: @@ -90,11 +97,12 @@ func pick(r rng, c *choice) node { // expand renders a template's compiled ops. compile validated every token, so this // cannot fail. -func expand(s *session, t *template) string { +func expand(s *session, t *template, shared *draws) string { var b strings.Builder b.Grow(t.grow) // One draw per held name, for this expansion only: a nested template and each - // repeat iteration get their own, since each is its own expansion. + // repeat iteration get their own, since each is its own expansion. A shared + // draw context, when a record supplies one, overrides that for references. var held *draws if len(t.held) > 0 { held = &draws{ @@ -108,7 +116,7 @@ func expand(s *session, t *template) string { case 'l': b.WriteString(o.lit) case 'f': - b.WriteString(readField(s, t, held, o.arms[s.IntN(len(o.arms))])) + b.WriteString(readField(s, t, held, shared, o.arms[s.IntN(len(o.arms))])) case 'b': // Read before the call, so the value a calc computes is the value the // format showed. calcVars fixed the order op.operands holds. @@ -116,7 +124,7 @@ func expand(s *session, t *template) string { if len(o.operands) > 0 { operands = make([]string, len(o.operands)) for j, a := range o.operands { - operands[j] = readField(s, t, held, a) + operands[j] = readField(s, t, held, shared, a) } } b.WriteString(o.call(s, b.String(), operands)) // b.String() is the output so far diff --git a/template_test.go b/template_test.go index 74dbc30..aa92a8a 100644 --- a/template_test.go +++ b/template_test.go @@ -315,7 +315,7 @@ func TestGrowIsALowerBound(t *testing.T) { t.Fatalf("format %q did not compile to a template", format) } for i := 0; i < 50; i++ { - if got := len(expand(f.rand, tmpl)); got < tmpl.grow { + if got := len(expand(f.rand, tmpl, nil)); got < tmpl.grow { t.Errorf("format %q: expand emitted %d bytes, below grow %d", format, got, tmpl.grow) } } -- 2.52.0 From 93ac610192cf5d27d6235ef0afe53ca60619c48b Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Thu, 3 Sep 2026 23:22:43 +0200 Subject: [PATCH 04/21] Reject field descent in Record and pin the shared-draw scope --- README.md | 15 +++++++++------ record.go | 10 +++++----- record_test.go | 31 +++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 835227b..35768d6 100644 --- a/README.md +++ b/README.md @@ -518,12 +518,15 @@ tokens add cost in proportion to the output. need a per-column `kind`, a parallel scalar system in a format whose promise is "text means what it says". A column's check digit, number or id is still valid-by-construction through a builtin; it is serialized as text. -- **A record's columns share one reference draw.** Two columns that reference one - category — `{/currency.code}` beside `{/currency.symbol}` — read one draw of it, - so a record's facts agree the way a template's [correlated - fields](#correlated-fields) do. Only references share across columns: a sibling - field is local to its own column, so `first` does not silently bind to a - `first` in the column next to it. +- **A record shares one reference draw per category.** Two columns that reference + one category — `{/currency.code}` beside `{/currency.symbol}` — read one draw of + it, so a record's facts agree the way a template's [correlated + fields](#correlated-fields) do. The draw is one per record, so it spans a + column's `repeat` and nested templates too (one record is one coherent unit); + a bare reference — `{/currency}`, no field — stays an independent draw every + time, the rule a format string already follows. Only references share: a sibling + field is local to its own column, so a `first` column does not silently bind to + a `first` in the column next to it. - **Filling a Go struct is out of scope.** `Record.Fields()` returns the columns a caller maps onto a struct themselves. gofakeit's `fake:"{firstname}"` tags reflect over an arbitrary struct type and cast into its fields — a different diff --git a/record.go b/record.go index 0b7d1d3..be485ba 100644 --- a/record.go +++ b/record.go @@ -88,17 +88,17 @@ func quoteIdent(s string) string { } // Record renders a path as one record: the template it names, with each direct -// field drawn as a column. A path naming a folder or a value that is not a -// template — a bare string or a choice, which have no fields — is an error. +// field drawn as a column. Only a category-level template is a record — a path +// that descends into a field, or that names a folder or a choice, is an error. func (f *Generator) Record(path string) (*Record, error) { f.mu.Lock() defer f.mu.Unlock() - n, err := descend(f.rand, &group{children: f.categories}, strings.Split(path, ".")) + _, n, tail, err := resolveRef(f.categories, strings.Split(path, ".")) if err != nil { return nil, fmt.Errorf("fejkdata: %s: %w", path, err) } - if _, ok := n.(*group); ok { - return nil, fmt.Errorf("fejkdata: %s names a folder, not a value", path) + if len(tail) > 0 { + return nil, fmt.Errorf("fejkdata: %s descends into %q, a field; only a category-level template is a record", path, tail[0]) } t, ok := n.(*template) if !ok { diff --git a/record_test.go b/record_test.go index 7493c88..8c0b638 100644 --- a/record_test.go +++ b/record_test.go @@ -139,6 +139,19 @@ func TestRecordSQLInsert(t *testing.T) { } } +func TestRecordRejectsFieldDescent(t *testing.T) { + dir := writeData(t, map[string]string{ + "cat": `{"format":"{sub}","sub":{"format":"{x}","x":"1"}}`, + "row": `[{"format":"{x}","x":"1"},{"format":"{x}","x":"2"}]`, + }) + f := newGenerator(t, dir, WithSeed(1)) + for _, path := range []string{"cat.sub", "row.x"} { + if _, err := f.Record(path); err == nil || !strings.Contains(err.Error(), "field") { + t.Errorf("Record(%q) = %v, want a 'descends into a field' error", path, err) + } + } +} + func TestRecordSharesAReferenceAcrossColumns(t *testing.T) { dir := writeData(t, map[string]string{ "currency": `[{"format":"{code}","code":"AUD","symbol":"$"},{"format":"{code}","code":"EUR","symbol":"€"}]`, @@ -169,6 +182,24 @@ func TestRecordSharesAReferenceAcrossColumns(t *testing.T) { } } +func TestRecordSharesAReferenceIntoAColumnRepeat(t *testing.T) { + dir := writeData(t, map[string]string{ + "currency": `[{"format":"{code}","code":"AUD"},{"format":"{code}","code":"EUR"}]`, + "order": `{"format":"","codes":{"format":"{/currency.code}","repeat":3,"separator":"-"}}`, + }) + f := newGenerator(t, dir, WithSeed(1)) + for i := 0; i < 50; i++ { + r, err := f.Record("order") + if err != nil { + t.Fatal(err) + } + parts := strings.Split(r.Fields()[0].Value, "-") + if len(parts) != 3 || parts[0] != parts[1] || parts[1] != parts[2] { + t.Fatalf("codes column = %q, want one shared draw across its repeat", r.Fields()[0].Value) + } + } +} + func TestRecordSQLQuotesIdentifiers(t *testing.T) { dir := writeData(t, map[string]string{ "row": `{"format": "", "postal-code": "1", "street-number": "2"}`, -- 2.52.0 From d237c15d778802cfaed5b63e158a2a34d16e5697 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Thu, 3 Sep 2026 23:29:10 +0200 Subject: [PATCH 05/21] Pin bare-reference independence and tighten the shared-draw comments --- hold.go | 3 +-- record_test.go | 26 ++++++++++++++++++++++++++ render.go | 3 +-- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/hold.go b/hold.go index a1eddb3..ef96774 100644 --- a/hold.go +++ b/hold.go @@ -274,8 +274,7 @@ func readField(s *session, t *template, held, shared *draws, a arm) string { } return renderShared(s, t.fields[a.key], shared) } - // A reference names a shared source, so a record shares its draw across the - // columns; a sibling field is drawn per expansion as always. + // A reference shares its draw across a record's columns; a sibling is per expansion. d := held if isRef(a.key) && shared != nil { d = shared diff --git a/record_test.go b/record_test.go index 8c0b638..472fb56 100644 --- a/record_test.go +++ b/record_test.go @@ -200,6 +200,32 @@ func TestRecordSharesAReferenceIntoAColumnRepeat(t *testing.T) { } } +func TestRecordBareReferenceStaysIndependent(t *testing.T) { + dir := writeData(t, map[string]string{ + "currency": `[{"format":"{code}","code":"AUD"},{"format":"{code}","code":"EUR"}]`, + "order": `{"format":"","whole":"{/currency}","code":"{/currency.code}"}`, + }) + f := newGenerator(t, dir, WithSeed(1)) + sawMismatch := false + for i := 0; i < 100; i++ { + r, err := f.Record("order") + if err != nil { + t.Fatal(err) + } + m := map[string]string{} + for _, c := range r.Fields() { + m[c.Name] = c.Value + } + if m["whole"] != m["code"] { + sawMismatch = true + break + } + } + if !sawMismatch { + t.Fatal("a bare {/currency} column never disagreed with a tailed {/currency.code} column; a bare reference should draw independently") + } +} + func TestRecordSQLQuotesIdentifiers(t *testing.T) { dir := writeData(t, map[string]string{ "row": `{"format": "", "postal-code": "1", "street-number": "2"}`, diff --git a/render.go b/render.go index b930cac..3dabd45 100644 --- a/render.go +++ b/render.go @@ -56,8 +56,7 @@ func render(s *session, n node) string { } // renderShared is render with a shared draw context: the draws a record shares -// across its columns. A nil shared means a standalone render, where a reference -// is drawn per expansion as it always has been. +// across its columns; a nil shared is a standalone render. func renderShared(s *session, n node, shared *draws) string { switch n := n.(type) { case *choice: -- 2.52.0 From ee9033783468d4b7a375518c7f91ac3d52591a32 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 4 Sep 2026 07:39:28 +0200 Subject: [PATCH 06/21] Fix the Records examples and pin their seeded output --- README.md | 36 +++++++++++++++++++----------------- cmd/fejkdata/main.go | 2 +- readme_test.go | 11 +++++++---- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 35768d6..37ed0be 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ brace, a bracket or a quote, so the two cannot collide (see | `-n`, `--repeat N` | render the value N times (up to 1048576), each an independent draw, streamed | | `--separator S` | between repeated values (default a newline) | | `--format F` | `text` (default), `json`, `csv` or `sql` — a record's columns, one per line | -| `--table T` | the INSERT target for `--format sql` (default: the path's last segment) | +| `--table T` | the INSERT target for `--format sql` (default: the path's last segment, or `records` for an inline template) | | `--list` | print every path, then exit | | `--version`, `-h`, `--help` | print, then exit | @@ -82,7 +82,9 @@ For structured output a record writes the row for you. A record is a template seen as columns: its fields are the columns, its `format` the whole. `--format json|csv|sql` streams a record per line; the library's -`Record` (below) hands back the columns. Save `mydata/users.json`: +`Record` (below) hands back the columns. Every column is a string — this is the +out-of-scope of typed scalars, see [Decisions](#decisions). Save +`mydata/users.json`: ```json { @@ -93,19 +95,19 @@ the whole. `--format json|csv|sql` streams a record per line; the library's ``` ```sh -fejkdata --format json users # {"first":"Ada","last":"Lovelace"} -fejkdata --format csv users # first,last → Ada,Lovelace -fejkdata --format sql users # INSERT INTO "users" ("first", "last") VALUES ('Ada', 'Lovelace'); -fejkdata --format json --repeat 3 users # three objects, one per line -fejkdata --format sql --table people users # INSERT into another table +fejkdata --seed 1 --data-path ./mydata --format json users # {"first":"Bo","last":"Lovelace"} +fejkdata --seed 1 --data-path ./mydata --format csv users # first,last → Bo,Lovelace +fejkdata --seed 1 --data-path ./mydata --format sql users # INSERT INTO "users" ("first", "last") VALUES ('Bo', 'Lovelace'); +fejkdata --seed 1 --data-path ./mydata --format json --repeat 3 users # three objects, one per line +fejkdata --seed 1 --data-path ./mydata --format sql --table people users # INSERT into another table ``` -`--repeat` streams that many records — a JSON object per line, a CSV row per -line after a header, an INSERT per line in SQL. Only a category-level template is -a record; a field, choice or folder errors. Column identifiers are double-quoted -in SQL, so a hyphenated field like `postal-code` stays valid. A column is a -string, and each format quotes it as such; see [Decisions](#decisions) for the -typed-scalar, correlation and struct-filling scope. +`--repeat` streams that many records — newline-delimited JSON (one object per +line, NDJSON), a CSV row per line after a header, or an INSERT per line in SQL. +To fold NDJSON into a single array, `fejkdata … --format json | jq -s .`. Only a +category-level template is a record; a field, choice or folder errors. Column +identifiers are double-quoted in SQL, so a hyphenated field like `postal-code` +stays valid. A record written only to emit columns still needs a `format` — the grammar's one required key — so `"format": ""` carries the fields with an inert format: it @@ -528,10 +530,10 @@ tokens add cost in proportion to the output. field is local to its own column, so a `first` column does not silently bind to a `first` in the column next to it. - **Filling a Go struct is out of scope.** `Record.Fields()` returns the columns a - caller maps onto a struct themselves. gofakeit's `fake:"{firstname}"` tags - reflect over an arbitrary struct type and cast into its fields — a different - concern from "data lives in JSON", and one a JSON record feeds without fejkdata - owning the reflection. + caller maps onto a struct themselves, casting each string to the field's type. + gofakeit's `fake:"{firstname}"` tags reflect over an arbitrary struct type and + cast into its fields — a different concern from "data lives in JSON", and one + whose typed casting fejkdata leaves to the caller rather than owning. - **The performance gate asserts allocations, not wall-clock time.** `AllocsPerRun` is deterministic across machines, so a ±10% ceiling does not flake under CI load, while time varies with the machine and its neighbours. A rendering slowdown diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index 3dbd5fb..0e6fbd9 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -46,7 +46,7 @@ row (after a header), or one INSERT. -n, --repeat N render the value N times, 1..1048576 (default 1) -s, --seed N seed for reproducible output --separator S string between repeated values (default newline) - --table T the INSERT target for --format sql (default: the path's last segment) + --table T the INSERT target for --format sql (default: the path's last segment, or records for an inline template) --version print the version, then exit Flags may come before or after ; -- ends the flags. A short flag's diff --git a/readme_test.go b/readme_test.go index 0a12f81..6bd4fec 100644 --- a/readme_test.go +++ b/readme_test.go @@ -60,11 +60,14 @@ func TestReadmeRecordExample(t *testing.T) { t.Fatal(err) } got := r.Fields() - if len(got) != 2 || got[0].Name != "first" || got[1].Name != "last" { - t.Fatalf("record columns = %v, want first, last", got) + if len(got) != 2 || got[0].Name != "first" || got[0].Value != "Bo" || got[1].Name != "last" || got[1].Value != "Lovelace" { + t.Fatalf("record columns = %v, want first=Bo, last=Lovelace with seed 1", got) } - if r.CSVHeader() != "first,last" || !strings.HasPrefix(r.SQLInsert("users"), `INSERT INTO "users" ("first", "last") VALUES (`) { - t.Fatalf("serializers = %q, %q, want first,last and an INSERT", r.CSVHeader(), r.SQLInsert("users")) + if r.CSVHeader() != "first,last" || r.CSVLine() != "Bo,Lovelace" { + t.Fatalf("csv = %q, %q, want first,last / Bo,Lovelace", r.CSVHeader(), r.CSVLine()) + } + if r.SQLInsert("users") != `INSERT INTO "users" ("first", "last") VALUES ('Bo', 'Lovelace');` { + t.Fatalf("sql = %q, want the seeded INSERT", r.SQLInsert("users")) } } -- 2.52.0 From 54e6865b1132155f76e80bf04d500eb3a37835f5 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 4 Sep 2026 08:38:03 +0200 Subject: [PATCH 07/21] Tests: columns as the record's public name, and a top-level repeat rejected at both record entry points --- cmd/fejkdata/main_test.go | 21 ++++++++++++++- readme_test.go | 2 +- record_test.go | 57 ++++++++++++++++++++++++++++----------- template_test.go | 2 +- 4 files changed, 63 insertions(+), 19 deletions(-) diff --git a/cmd/fejkdata/main_test.go b/cmd/fejkdata/main_test.go index d56625a..a97ba54 100644 --- a/cmd/fejkdata/main_test.go +++ b/cmd/fejkdata/main_test.go @@ -531,7 +531,7 @@ func TestRunRecordMisuse(t *testing.T) { args []string want string }{ - {[]string{"--format", "yaml", "users"}, "--format takes text, json, csv or sql"}, + {[]string{"--format", "yaml", "users"}, "--format takes csv, json, sql or text"}, {[]string{"--format", "json", "--separator", ",", "users"}, "--separator joins text values"}, {[]string{"--table", "t", "users"}, "--table names the INSERT target"}, {[]string{"--format", "json", "--table", "t", "users"}, "--table names the INSERT target"}, @@ -554,6 +554,25 @@ func TestRunRecordOnABareValueIsRuntimeError(t *testing.T) { } } +func TestRunRecordRejectsATopLevelRepeat(t *testing.T) { + dir := t.TempDir() + content := `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}` + if err := os.WriteFile(filepath.Join(dir, "rep.json"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + code, _, errb := runOut("--format", "json", "--data-path", dir, "rep") + if code != 1 || !strings.Contains(errb, "carries repeat 3") { + t.Errorf("record on a repeating path = %d, %q; want exit 1 naming the repeat", code, errb) + } + code, _, errb = runOut("--format", "json", content) + if code != 2 || !strings.Contains(errb, "carries repeat 3") { + t.Errorf("record on a repeating inline template = %d, %q; want exit 2 naming the repeat", code, errb) + } + if code, out, _ := runOut("--seed", "1", "--data-path", dir, "rep"); code != 0 || strings.TrimSpace(out) != "x-|x-|x-" { + t.Errorf("text view = %d, %q; want the repeat still composed", code, out) + } +} + func TestRunNumbersFollowTheShell(t *testing.T) { _, want, _ := runOut("--seed", "7", "--repeat", "3", "sv_SE.word") code, got, errb := runOut("--seed", "007", "--repeat", "+3", "sv_SE.word") diff --git a/readme_test.go b/readme_test.go index 6bd4fec..4cc569e 100644 --- a/readme_test.go +++ b/readme_test.go @@ -59,7 +59,7 @@ func TestReadmeRecordExample(t *testing.T) { if err != nil { t.Fatal(err) } - got := r.Fields() + got := r.Columns() if len(got) != 2 || got[0].Name != "first" || got[0].Value != "Bo" || got[1].Name != "last" || got[1].Value != "Lovelace" { t.Fatalf("record columns = %v, want first=Bo, last=Lovelace with seed 1", got) } diff --git a/record_test.go b/record_test.go index 472fb56..ab38a2a 100644 --- a/record_test.go +++ b/record_test.go @@ -25,9 +25,9 @@ func TestRecordProjectsFieldsAsColumns(t *testing.T) { if err != nil { t.Fatal(err) } - got := r.Fields() + got := r.Columns() if len(got) != 2 || got[0].Name != "first" || got[1].Name != "last" { - t.Fatalf("Record.Fields() = %v, want columns first, last in name order", got) + t.Fatalf("Record.Columns() = %v, want columns first, last in name order", got) } if (got[0].Value != "Ada" && got[0].Value != "Bo") || (got[1].Value != "Lovelace" && got[1].Value != "Ek") { t.Fatalf("columns = %v, want the field values", got) @@ -55,9 +55,9 @@ func TestRecordSkipsReferenceBindings(t *testing.T) { if err != nil { t.Fatal(err) } - got := r.Fields() + got := r.Columns() if len(got) != 1 || got[0].Name != "id" { - t.Fatalf("Record.Fields() = %v, want only the id column (a {/path} is not a column)", got) + t.Fatalf("Record.Columns() = %v, want only the id column (a {/path} is not a column)", got) } } @@ -96,7 +96,7 @@ func TestRecordJSON(t *testing.T) { if len(m) != 2 || (m["first"] != "Ada" && m["first"] != "Bo") { t.Fatalf("Record.JSON() = %s, want two addressable columns", r.JSON()) } - for _, f := range r.Fields() { + for _, f := range r.Columns() { if m[f.Name] != f.Value { t.Fatalf("JSON column %q = %q, want %q", f.Name, m[f.Name], f.Value) } @@ -119,7 +119,7 @@ func TestRecordCSV(t *testing.T) { if err != nil { t.Fatalf("CSVLine() is not valid CSV: %v\n%q", err, r.CSVLine()) } - if len(rec) != 1 || rec[0] != r.Fields()[0].Value { + if len(rec) != 1 || rec[0] != r.Columns()[0].Value { t.Fatalf("CSVLine() = %v, want the field value round-tripped", rec) } } @@ -164,7 +164,7 @@ func TestRecordSharesAReferenceAcrossColumns(t *testing.T) { t.Fatal(err) } m := map[string]string{} - for _, c := range r.Fields() { + for _, c := range r.Columns() { m[c.Name] = c.Value } switch m["code"] { @@ -193,9 +193,9 @@ func TestRecordSharesAReferenceIntoAColumnRepeat(t *testing.T) { if err != nil { t.Fatal(err) } - parts := strings.Split(r.Fields()[0].Value, "-") + parts := strings.Split(r.Columns()[0].Value, "-") if len(parts) != 3 || parts[0] != parts[1] || parts[1] != parts[2] { - t.Fatalf("codes column = %q, want one shared draw across its repeat", r.Fields()[0].Value) + t.Fatalf("codes column = %q, want one shared draw across its repeat", r.Columns()[0].Value) } } } @@ -213,7 +213,7 @@ func TestRecordBareReferenceStaysIndependent(t *testing.T) { t.Fatal(err) } m := map[string]string{} - for _, c := range r.Fields() { + for _, c := range r.Columns() { m[c.Name] = c.Value } if m["whole"] != m["code"] { @@ -248,16 +248,16 @@ func TestFakeRecordAndTemplate(t *testing.T) { if err != nil { t.Fatalf("FakeRecord: %v", err) } - if len(want.Fields()) != 2 { - t.Fatalf("FakeRecord fields = %v, want two columns", want.Fields()) + if len(want.Columns()) != 2 { + t.Fatalf("FakeRecord columns = %v, want two columns", want.Columns()) } reusable, err := f.NewRecordTemplate(in) if err != nil { t.Fatalf("NewRecordTemplate: %v", err) } for i := 0; i < 20; i++ { - if got := reusable.Fake(); len(got.Fields()) != 2 { - t.Fatalf("RecordTemplate.Fake() = %v, want two columns", got.Fields()) + if got := reusable.Fake(); len(got.Columns()) != 2 { + t.Fatalf("RecordTemplate.Fake() = %v, want two columns", got.Columns()) } } } @@ -268,11 +268,36 @@ func TestInlineRecordErrors(t *testing.T) { input string want string }{ - {`"hello"`, "needs at least one field"}, - {`["a","b"]`, "a choice"}, + {`"hello"`, "has no fields, so no columns"}, + {`["a","b"]`, "names a choice, not a template"}, + {`{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`, "carries repeat 3"}, } { if _, err := f.FakeRecord(c.input); err == nil || !strings.Contains(err.Error(), c.want) { t.Errorf("FakeRecord(%q) = %v, want an error naming %q", c.input, err, c.want) } } } + +func TestRecordRejectsATopLevelRepeat(t *testing.T) { + dir := writeData(t, map[string]string{ + "rep": `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`, + }) + f := newGenerator(t, dir, WithSeed(1)) + if _, err := f.Record("rep"); err == nil || !strings.Contains(err.Error(), "carries repeat 3") { + t.Errorf("Record on a repeating template = %v, want an error naming the repeat", err) + } + if v, err := f.Fake("rep"); err != nil || v != "x-|x-|x-" { + t.Errorf("Fake(rep) = %q, %v, want the repeat still composed for the string view", v, err) + } +} + +func TestRecordTemplateRejectsATopLevelRepeat(t *testing.T) { + f := recordCat(t) + in := `{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}` + if _, err := f.NewRecordTemplate(in); err == nil || !strings.Contains(err.Error(), "carries repeat 3") { + t.Errorf("NewRecordTemplate on a repeating template = %v, want an error naming the repeat", err) + } + if _, err := f.NewTemplate(in); err != nil { + t.Errorf("NewTemplate on the same input = %v, want the string view to still compile", err) + } +} diff --git a/template_test.go b/template_test.go index aa92a8a..080972e 100644 --- a/template_test.go +++ b/template_test.go @@ -38,7 +38,7 @@ func compiled(t *testing.T, s string) node { func mustRender(t *testing.T, f *Generator, s string) string { t.Helper() - return render(f.rand, compiled(t, s)) + return render(f.rand, compiled(t, s), nil) } func TestStringIsAFormat(t *testing.T) { -- 2.52.0 From 8cb8cf6334e6161225921ab438c3d01b17fa625e Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 4 Sep 2026 08:40:58 +0200 Subject: [PATCH 08/21] Reject a record's top-level repeat, share one record fence between both entry points, and give the renderer one spelling --- cmd/fejkdata/main.go | 107 ++++++++++++++++++++++++----------------- hold.go | 13 ++--- inline.go | 2 +- record.go | 110 ++++++++++++++++++++++--------------------- reference.go | 12 ++--- render.go | 29 +++++------- 6 files changed, 146 insertions(+), 127 deletions(-) diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index 0e6fbd9..c3163fb 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -15,6 +15,7 @@ import ( "io" "os" "runtime/debug" + "sort" "strconv" "strings" @@ -208,20 +209,41 @@ func parseArgs(argv []string) (invocation, error) { return in, nil } -// recordFormat reports whether the format names a record output (json, csv or -// sql) rather than the default text. +// recordFormat is one way to write a record out: the line it renders, and the +// header that precedes the first one, if the format has one. +type recordFormat struct { + header func(*fejkdata.Record) string + line func(r *fejkdata.Record, table string) string +} + +// recordFormats is every --format that writes records. Adding one is this entry +// alone: the flag check reads the same table the writer dispatches through, so a +// format cannot be accepted and then not written. +var recordFormats = map[string]recordFormat{ + "csv": {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }}, + "json": {line: func(r *fejkdata.Record, _ string) string { return r.JSON() }}, + "sql": {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }}, +} + +// recordFormat reports whether the format writes records rather than plain text. func (in invocation) recordFormat() bool { - return in.formatSet && in.format != "text" + return in.format != "text" +} + +// formatNames lists the --format values, text included, for the misuse error. +func formatNames() string { + names := []string{"text"} + for name := range recordFormats { + names = append(names, name) + } + sort.Strings(names) + return strings.Join(names[:len(names)-1], ", ") + " or " + names[len(names)-1] } // checkFlags rejects a flag value or combination that cannot run. func (in invocation) checkFlags() error { - if in.formatSet { - switch in.format { - case "text", "json", "csv", "sql": - default: - return fmt.Errorf("--format takes text, json, csv or sql, got %q", in.format) - } + if _, ok := recordFormats[in.format]; !ok && in.format != "text" { + return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format) } if in.tableSet && in.format != "sql" { return errors.New("--table names the INSERT target, so it needs --format sql") @@ -272,17 +294,13 @@ func (in invocation) options() []fejkdata.Option { // so a render failure comes before anything is written; a write failure surfaces // from Flush, bufio keeping the first one. func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) error { - arg := in.paths[0] - if in.recordFormat() { - return in.writeRecords(f, kind, arg, w) + draw, err := in.draw(f, kind, in.paths[0]) + if err != nil { + return err } - draw := func() (string, error) { return f.Fake(arg) } - if kind == argTemplate { - t, err := f.NewTemplate(arg) - if err != nil { - return templateError{err} - } - draw = func() (string, error) { return t.Fake(), nil } + separator := in.separator + if in.recordFormat() { + separator = "\n" } out := bufio.NewWriter(w) for i := 0; i < in.repeat; i++ { @@ -291,7 +309,7 @@ func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) err return err } if i > 0 { - out.WriteString(in.separator) + out.WriteString(separator) } out.WriteString(v) } @@ -299,40 +317,43 @@ func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) err return out.Flush() } -// writeRecords streams a record per line in the chosen format: one JSON object -// per line, a CSV header then a row per line, or one INSERT per line. -func (in invocation) writeRecords(f *fejkdata.Generator, kind argKind, arg string, w io.Writer) error { - draw := func() (*fejkdata.Record, error) { return f.Record(arg) } +// draw builds what one render yields: the value's text, or the record's line in +// the chosen format, the header carried ahead of the first one. +func (in invocation) draw(f *fejkdata.Generator, kind argKind, arg string) (func() (string, error), error) { + if !in.recordFormat() { + if kind != argTemplate { + return func() (string, error) { return f.Fake(arg) }, nil + } + t, err := f.NewTemplate(arg) + if err != nil { + return nil, templateError{err} + } + return func() (string, error) { return t.Fake(), nil }, nil + } + record := func() (*fejkdata.Record, error) { return f.Record(arg) } if kind == argTemplate { t, err := f.NewRecordTemplate(arg) if err != nil { - return templateError{err} + return nil, templateError{err} } - draw = func() (*fejkdata.Record, error) { return t.Fake(), nil } + record = func() (*fejkdata.Record, error) { return t.Fake(), nil } } - out := bufio.NewWriter(w) - table := in.table + format, table, first := recordFormats[in.format], in.table, true if table == "" { table = defaultTable(arg, kind) } - for i := 0; i < in.repeat; i++ { - r, err := draw() + return func() (string, error) { + r, err := record() if err != nil { - return err + return "", err } - switch in.format { - case "json": - fmt.Fprintln(out, r.JSON()) - case "csv": - if i == 0 { - fmt.Fprintln(out, r.CSVHeader()) - } - fmt.Fprintln(out, r.CSVLine()) - case "sql": - fmt.Fprintln(out, r.SQLInsert(table)) + line := format.line(r, table) + if first && format.header != nil { + line = format.header(r) + "\n" + line } - } - return out.Flush() + first = false + return line, nil + }, nil } // defaultTable names the INSERT target when --table is absent: the path's last diff --git a/hold.go b/hold.go index ef96774..0c7d6b5 100644 --- a/hold.go +++ b/hold.go @@ -267,17 +267,18 @@ type draws struct { // gives one value, and a shown operand is the operand computed. Every other name is // drawn afresh, so {word} {word} still draws twice. checkTokens, checkPath and // linkRefs prove every step, so the walk cannot fail. -func readField(s *session, t *template, held, shared *draws, a arm) string { +func readField(s *session, t *template, held, refScope *draws, a arm) string { if !t.held[a.key] { if len(a.tail) > 0 { panic(fmt.Sprintf("fejkdata: %q reads a path into %q, which the expansion does not hold", a.name, a.key)) } - return renderShared(s, t.fields[a.key], shared) + return render(s, t.fields[a.key], refScope) } - // A reference shares its draw across a record's columns; a sibling is per expansion. + // A reference reads the caller's scope, so its draw outlives this expansion; a + // sibling stays local to it. d := held - if isRef(a.key) && shared != nil { - d = shared + if isRef(a.key) && refScope != nil { + d = refScope } if v, read := d.value[a.path]; read { return v @@ -298,7 +299,7 @@ func readField(s *session, t *template, held, shared *draws, a arm) string { } return []node{n}, nil }, - leaf: func(n node) error { v = renderShared(s, n, shared); return nil }, + leaf: func(n node) error { v = render(s, n, refScope); return nil }, }) d.value[a.path] = v return v diff --git a/inline.go b/inline.go index f68ca29..91f6686 100644 --- a/inline.go +++ b/inline.go @@ -20,7 +20,7 @@ type Template struct { func (t *Template) Fake() string { t.g.mu.Lock() defer t.g.mu.Unlock() - return render(t.g.rand, t.n) + return render(t.g.rand, t.n, nil) } // NewTemplate compiles an inline template — a format string or a JSON value — and diff --git a/record.go b/record.go index be485ba..3eadb16 100644 --- a/record.go +++ b/record.go @@ -3,13 +3,13 @@ package fejkdata import ( "encoding/csv" "encoding/json" + "errors" "fmt" - "sort" "strings" ) -// Field is one rendered column of a record. -type Field struct { +// Column is one rendered column of a record. +type Column struct { Name string Value string } @@ -19,19 +19,19 @@ type Field struct { // string a [Generator.Fake] call renders; a record is its inverse — each field // projected as a column instead of composed. type Record struct { - fields []Field + columns []Column } -// Fields returns the record's columns in name order. -func (r *Record) Fields() []Field { - return append([]Field(nil), r.fields...) +// Columns returns the record's columns in name order. +func (r *Record) Columns() []Column { + return append([]Column(nil), r.columns...) } // JSON renders the record as one JSON object, every column a string. func (r *Record) JSON() string { - m := make(map[string]string, len(r.fields)) - for _, f := range r.fields { - m[f.Name] = f.Value + m := make(map[string]string, len(r.columns)) + for _, c := range r.columns { + m[c.Name] = c.Value } b, _ := json.Marshal(m) return string(b) @@ -48,17 +48,17 @@ func (r *Record) CSVLine() string { } func (r *Record) names() []string { - out := make([]string, len(r.fields)) - for i, f := range r.fields { - out[i] = f.Name + out := make([]string, len(r.columns)) + for i, c := range r.columns { + out[i] = c.Name } return out } func (r *Record) values() []string { - out := make([]string, len(r.fields)) - for i, f := range r.fields { - out[i] = f.Value + out := make([]string, len(r.columns)) + for i, c := range r.columns { + out[i] = c.Value } return out } @@ -74,11 +74,11 @@ func csvLine(cols []string) string { // SQLInsert renders the record as one INSERT statement into table: identifiers in // ANSI double quotes, every value a single-quoted string literal. func (r *Record) SQLInsert(table string) string { - cols := make([]string, len(r.fields)) - vals := make([]string, len(r.fields)) - for i, f := range r.fields { - cols[i] = quoteIdent(f.Name) - vals[i] = "'" + strings.ReplaceAll(f.Value, "'", "''") + "'" + cols := make([]string, len(r.columns)) + vals := make([]string, len(r.columns)) + for i, c := range r.columns { + cols[i] = quoteIdent(c.Name) + vals[i] = "'" + strings.ReplaceAll(c.Value, "'", "''") + "'" } return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);", quoteIdent(table), strings.Join(cols, ", "), strings.Join(vals, ", ")) } @@ -93,60 +93,46 @@ func quoteIdent(s string) string { func (f *Generator) Record(path string) (*Record, error) { f.mu.Lock() defer f.mu.Unlock() - _, n, tail, err := resolveRef(f.categories, strings.Split(path, ".")) + _, n, tail, err := resolveCategory(f.categories, strings.Split(path, ".")) if err != nil { return nil, fmt.Errorf("fejkdata: %s: %w", path, err) } if len(tail) > 0 { return nil, fmt.Errorf("fejkdata: %s descends into %q, a field; only a category-level template is a record", path, tail[0]) } - t, ok := n.(*template) - if !ok { - return nil, fmt.Errorf("fejkdata: %s names a choice, not a template; only a category-level template is a record", path) + t, err := recordOf(n) + if err != nil { + return nil, fmt.Errorf("fejkdata: %s %w", path, err) } - r := renderRecord(f.rand, t) - if len(r.fields) == 0 { - return nil, fmt.Errorf("fejkdata: %s has no fields, so no columns", path) - } - return r, nil + return renderRecord(f.rand, t), nil } // RecordTemplate is an inline record compiled, referenced and validated once, // ready to render many times with [RecordTemplate.Fake]. type RecordTemplate struct { g *Generator - n node + t *template } // Fake renders the record with one draw. func (t *RecordTemplate) Fake() *Record { t.g.mu.Lock() defer t.g.mu.Unlock() - return renderRecord(t.g.rand, t.n.(*template)) + return renderRecord(t.g.rand, t.t) } // NewRecordTemplate compiles an inline record — a JSON object with a format and // fields — and binds its references against the loaded tree. func (f *Generator) NewRecordTemplate(input string) (*RecordTemplate, error) { - n, err := compileInput(input) + t, err := f.NewTemplate(input) if err != nil { - return nil, fmt.Errorf("fejkdata: %w", err) + return nil, err } - t, ok := n.(*template) - if !ok { - return nil, fmt.Errorf("fejkdata: an inline record is a JSON object with a format and fields, not a choice") + rt, err := recordOf(t.n) + if err != nil { + return nil, fmt.Errorf("fejkdata: an inline record %w", err) } - if len(recordColumns(t)) == 0 { - return nil, fmt.Errorf("fejkdata: an inline record needs at least one field to project as a column") - } - scope := inlineScope(t) - if err := linkNodeRefs(scope, f.categories); err != nil { - return nil, fmt.Errorf("fejkdata: %w", err) - } - if err := checkScope(scope); err != nil { - return nil, fmt.Errorf("fejkdata: %w", err) - } - return &RecordTemplate{g: f, n: t}, nil + return &RecordTemplate{g: f, t: rt}, nil } // FakeRecord compiles and renders an inline record in one call. @@ -158,15 +144,32 @@ func (f *Generator) FakeRecord(input string) (*Record, error) { return t.Fake(), nil } +// recordOf is the fence both record entry points pass: the node is a +// category-level template, it carries no repeat — which composes the format +// rather than projecting columns — and it offers at least one column. +func recordOf(n node) (*template, error) { + t, ok := n.(*template) + if !ok { + return nil, errors.New("names a choice, not a template; only a category-level template is a record") + } + if t.repeat != 1 { + return nil, fmt.Errorf("carries repeat %d, which composes its format into one string; a record projects columns instead — drop the repeat and render the record again for more rows", t.repeat) + } + if len(recordColumns(t)) == 0 { + return nil, errors.New("has no fields, so no columns") + } + return t, nil +} + // renderRecord projects a template's direct fields as columns, drawn once each, // in name order. A {/path} binding is a render edge, not a column, so it is -// skipped the same way List and the graph do. The columns share one draw context, -// so two columns that reference one category read one draw of it. +// skipped the same way List and the graph do. The columns share one reference +// scope, so two columns that reference one category read one draw of it. func renderRecord(s *session, t *template) *Record { - shared := &draws{variant: map[string]node{}, value: map[string]string{}} + scope := &draws{variant: map[string]node{}, value: map[string]string{}} r := &Record{} for _, name := range recordColumns(t) { - r.fields = append(r.fields, Field{Name: name, Value: renderShared(s, t.fields[name], shared)}) + r.columns = append(r.columns, Column{Name: name, Value: render(s, t.fields[name], scope)}) } return r } @@ -176,11 +179,10 @@ func renderRecord(s *session, t *template) *Record { // name that is not a reference is a column. func recordColumns(t *template) []string { var names []string - for name := range t.fields { + for _, name := range sortedNames(t.fields) { if !isRef(name) { names = append(names, name) } } - sort.Strings(names) return names } diff --git a/reference.go b/reference.go index 7744474..ff26770 100644 --- a/reference.go +++ b/reference.go @@ -93,7 +93,7 @@ func linkTemplateRefs(folder []string, path string, t *template, root map[string if err != nil { return fmt.Errorf("%s: reference {%s}: %w", path, name, err) } - head, target, tail, err := resolveRef(root, segments) + head, target, tail, err := resolveCategory(root, segments) if err != nil { return fmt.Errorf("%s: reference {%s}: %w", path, name, err) } @@ -146,11 +146,11 @@ func eachTemplate(root map[string]node, fn func(folder []string, path string, t return inFolder(nil, root) } -// resolveRef walks a reference path through the folders to the category it names, -// returning that head, the node, and the tail left to read into it. A descent of -// its own rather than a walkPath: it walks groups only and returns where they end, -// not a leaf. -func resolveRef(root map[string]node, segments []string) (head []string, target node, tail []string, err error) { +// resolveCategory walks a dotted path through the folders to the category it +// names, returning that head, the node, and the tail left to read into it. A +// descent of its own rather than a walkPath: it walks groups only and returns +// where they end, not a leaf. +func resolveCategory(root map[string]node, segments []string) (head []string, target node, tail []string, err error) { var n node = &group{children: root} i := 0 for ; i < len(segments); i++ { diff --git a/render.go b/render.go index 3dabd45..7a7af3a 100644 --- a/render.go +++ b/render.go @@ -27,7 +27,7 @@ func (f *Generator) Fake(path string) (string, error) { if _, ok := n.(*group); ok { return "", fmt.Errorf("fejkdata: %s names a folder, not a value", path) } - return render(f.rand, n), nil + return render(f.rand, n, nil), nil } // descend walks named fields to the node a path names. It is the one render-side @@ -50,23 +50,18 @@ func descend(s *session, root node, segments []string) (node, error) { } // render evaluates a compiled node to a string. compile validates every node up -// front, so rendering a compiled tree cannot fail. -func render(s *session, n node) string { - return renderShared(s, n, nil) -} - -// renderShared is render with a shared draw context: the draws a record shares -// across its columns; a nil shared is a standalone render. -func renderShared(s *session, n node, shared *draws) string { +// front, so rendering a compiled tree cannot fail. refScope carries the draws a +// reference shares beyond its own expansion; nil keeps every reference local. +func render(s *session, n node, refScope *draws) string { switch n := n.(type) { case *choice: - return renderShared(s, pick(s, n), shared) + return render(s, pick(s, n), refScope) case *template: if n.repeat == 1 { if n.fixed { return n.lit } - return expand(s, n, shared) + return expand(s, n, refScope) } var b strings.Builder b.Grow(n.repeat * (n.grow + len(n.separator))) @@ -74,7 +69,7 @@ func renderShared(s *session, n node, shared *draws) string { if i > 0 { b.WriteString(n.separator) } - b.WriteString(expand(s, n, shared)) + b.WriteString(expand(s, n, refScope)) } return b.String() default: @@ -96,12 +91,12 @@ func pick(r rng, c *choice) node { // expand renders a template's compiled ops. compile validated every token, so this // cannot fail. -func expand(s *session, t *template, shared *draws) string { +func expand(s *session, t *template, refScope *draws) string { var b strings.Builder b.Grow(t.grow) // One draw per held name, for this expansion only: a nested template and each - // repeat iteration get their own, since each is its own expansion. A shared - // draw context, when a record supplies one, overrides that for references. + // repeat iteration get their own, since each is its own expansion. A reference + // reads the caller's scope instead, whenever one was supplied. var held *draws if len(t.held) > 0 { held = &draws{ @@ -115,7 +110,7 @@ func expand(s *session, t *template, shared *draws) string { case 'l': b.WriteString(o.lit) case 'f': - b.WriteString(readField(s, t, held, shared, o.arms[s.IntN(len(o.arms))])) + b.WriteString(readField(s, t, held, refScope, o.arms[s.IntN(len(o.arms))])) case 'b': // Read before the call, so the value a calc computes is the value the // format showed. calcVars fixed the order op.operands holds. @@ -123,7 +118,7 @@ func expand(s *session, t *template, shared *draws) string { if len(o.operands) > 0 { operands = make([]string, len(o.operands)) for j, a := range o.operands { - operands[j] = readField(s, t, held, shared, a) + operands[j] = readField(s, t, held, refScope, a) } } b.WriteString(o.call(s, b.String(), operands)) // b.String() is the output so far -- 2.52.0 From 7ee7f1788b4999f18d84c8ccc49cd9fe7f5da514 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 4 Sep 2026 08:43:12 +0200 Subject: [PATCH 09/21] Record the SQL dialect, the newline-in-a-column shape, the fixed column set and the string view's divergence --- AGENTS.md | 7 +++++++ README.md | 45 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 75d511e..e5ecb93 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,6 +11,13 @@ # Deferred +- **A record cannot ask for an independent reference draw (2026-09-04).** Within + one record every tailed reference to a category is one draw, and a bare + `{/cat}` renders the whole category, so a column wanting its own draw of + `{/cat.field}` has no spelling for it. Left until a use case names which columns + should disagree; premise: a record is one coherent row, which is what columns + are for. Not raised in review before that. + - **Shipped-data de-duplication (2026-09-02).** `email.json`'s `local` is a drifted copy of `username.json`, and no shipped file yet uses a held path, an operand or a reference. Fixed in the data fill before the first tag, when the diff --git a/README.md b/README.md index 37ed0be..cb920a3 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ brace, a bracket or a quote, so the two cannot collide (see | `-s`, `--seed N` | reproducible output | | `-n`, `--repeat N` | render the value N times (up to 1048576), each an independent draw, streamed | | `--separator S` | between repeated values (default a newline) | -| `--format F` | `text` (default), `json`, `csv` or `sql` — a record's columns, one per line | +| `--format F` | `text` (default), `json`, `csv` or `sql` — a record's columns, one record per row | | `--table T` | the INSERT target for `--format sql` (default: the path's last segment, or `records` for an inline template) | | `--list` | print every path, then exit | | `--version`, `-h`, `--help` | print, then exit | @@ -81,7 +81,7 @@ For structured output a record writes the row for you. ### Records A record is a template seen as columns: its fields are the columns, its `format` -the whole. `--format json|csv|sql` streams a record per line; the library's +the whole. `--format json|csv|sql` streams one record per row; the library's `Record` (below) hands back the columns. Every column is a string — this is the out-of-scope of typed scalars, see [Decisions](#decisions). Save `mydata/users.json`: @@ -103,11 +103,22 @@ fejkdata --seed 1 --data-path ./mydata --format sql --table people users # INSER ``` `--repeat` streams that many records — newline-delimited JSON (one object per -line, NDJSON), a CSV row per line after a header, or an INSERT per line in SQL. -To fold NDJSON into a single array, `fejkdata … --format json | jq -s .`. Only a -category-level template is a record; a field, choice or folder errors. Column -identifiers are double-quoted in SQL, so a hyphenated field like `postal-code` -stays valid. +line, NDJSON), a CSV row after a header, or an INSERT in SQL. To fold NDJSON into +a single array, `fejkdata … --format json | jq -s .`. Only a category-level +template is a record; a field, choice or folder errors, and so does a `repeat` on +the template itself, which composes the format into one string rather than +projecting columns — ask for more records with `--repeat`. A `repeat` on a column +is fine. + +A column carrying a newline keeps it inside the quoted CSV field or the SQL string +literal, so a row can span physical lines: read the stream with a CSV or SQL +parser rather than splitting it on newlines. + +The SQL is ANSI — identifiers in double quotes, a literal quote doubled (`''`), +backslashes passed through — so a hyphenated field like `postal-code` stays a +valid identifier. PostgreSQL and SQLite take it as written; MySQL and MariaDB need +`ANSI_QUOTES` and `NO_BACKSLASH_ESCAPES` set first, or they read `"users"` as a +string and a backslash as an escape. A record written only to emit columns still needs a `format` — the grammar's one required key — so `"format": ""` carries the fields with an inert format: it @@ -146,7 +157,7 @@ r, err = f.FakeRecord(`{"format":"{x}","x":["a","b"]}`) // compile + render inli | `WithDataFS(fsys)` | layer an `fs.FS`, such as your own `embed.FS` | | `WithoutShippedData()` | load only what you give | -A `*Record` carries its columns via `Fields()`, and serializes them with `JSON()` +A `*Record` carries its columns via `Columns()`, and serializes them with `JSON()` (one object), `CSVHeader()`/`CSVLine()`, or `SQLInsert(table)` — the same three shapes the CLI's `--format` streams. `Record` and `FakeRecord` take a record; a path or template that is not one — a bare string, a choice, or a folder — errors. @@ -529,7 +540,23 @@ tokens add cost in proportion to the output. time, the rule a format string already follows. Only references share: a sibling field is local to its own column, so a `first` column does not silently bind to a `first` in the column next to it. -- **Filling a Go struct is out of scope.** `Record.Fields()` returns the columns a + + The string view of that same template does not share. `Fake` renders each + sibling field as its own expansion, so a `{/currency.code}` field beside a + `{/currency.symbol}` field is two draws and may render `EUR $`; writing both + references in one `format` holds them together, as + [One draw, one spelling](#one-draw-one-spelling) says. The scope is what makes a + row coherent when the columns *are* the output, and there the caller cannot fall + back on one format string. Widening it to every render would change what `Fake` + has emitted since the start, for a correlation a single format already reaches. +- **A record's column set is fixed before the first draw.** Only a category-level + template is a record: a path descending into a field, or naming a folder or a + choice, errors. A tail may pass through a choice whose variants carry different + fields, so the columns — and with them the CSV header written once ahead of every + row — would vary per draw. A fixed column set is what the CSV and `INSERT` + contracts rest on, so the restriction holds even where a particular choice would + happen to agree. +- **Filling a Go struct is out of scope.** `Record.Columns()` returns the columns a caller maps onto a struct themselves, casting each string to the field's type. gofakeit's `fake:"{firstname}"` tags reflect over an arbitrary struct type and cast into its fields — a different concern from "data lives in JSON", and one -- 2.52.0 From 59159d4f5dec205f6c8c94b8ed8168ab0bd118fe Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 4 Sep 2026 08:51:00 +0200 Subject: [PATCH 10/21] Tests: the record fence reads correctly for an inline record too --- record_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/record_test.go b/record_test.go index ab38a2a..e8226f4 100644 --- a/record_test.go +++ b/record_test.go @@ -269,7 +269,7 @@ func TestInlineRecordErrors(t *testing.T) { want string }{ {`"hello"`, "has no fields, so no columns"}, - {`["a","b"]`, "names a choice, not a template"}, + {`["a","b"]`, "a record is a template whose fields are its columns"}, {`{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`, "carries repeat 3"}, } { if _, err := f.FakeRecord(c.input); err == nil || !strings.Contains(err.Error(), c.want) { -- 2.52.0 From 83e11074ac62c9f70fbada92ea53782206b29875 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 4 Sep 2026 08:52:08 +0200 Subject: [PATCH 11/21] Fix the column set once, drop the unreachable separator override, and name the record predicate apart from the format table --- cmd/fejkdata/main.go | 14 +++++-------- record.go | 50 +++++++++++++++++++++++--------------------- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index c3163fb..17531f1 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -225,8 +225,8 @@ var recordFormats = map[string]recordFormat{ "sql": {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }}, } -// recordFormat reports whether the format writes records rather than plain text. -func (in invocation) recordFormat() bool { +// writesRecords reports whether the format writes records rather than plain text. +func (in invocation) writesRecords() bool { return in.format != "text" } @@ -248,7 +248,7 @@ func (in invocation) checkFlags() error { if in.tableSet && in.format != "sql" { return errors.New("--table names the INSERT target, so it needs --format sql") } - if in.recordFormat() && in.separatorSet { + if in.writesRecords() && in.separatorSet { return errors.New("--separator joins text values, so it has no effect with --format " + in.format) } return nil @@ -298,10 +298,6 @@ func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) err if err != nil { return err } - separator := in.separator - if in.recordFormat() { - separator = "\n" - } out := bufio.NewWriter(w) for i := 0; i < in.repeat; i++ { v, err := draw() @@ -309,7 +305,7 @@ func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) err return err } if i > 0 { - out.WriteString(separator) + out.WriteString(in.separator) } out.WriteString(v) } @@ -320,7 +316,7 @@ func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) err // draw builds what one render yields: the value's text, or the record's line in // the chosen format, the header carried ahead of the first one. func (in invocation) draw(f *fejkdata.Generator, kind argKind, arg string) (func() (string, error), error) { - if !in.recordFormat() { + if !in.writesRecords() { if kind != argTemplate { return func() (string, error) { return f.Fake(arg) }, nil } diff --git a/record.go b/record.go index 3eadb16..f4ce75a 100644 --- a/record.go +++ b/record.go @@ -100,25 +100,26 @@ func (f *Generator) Record(path string) (*Record, error) { if len(tail) > 0 { return nil, fmt.Errorf("fejkdata: %s descends into %q, a field; only a category-level template is a record", path, tail[0]) } - t, err := recordOf(n) + t, columns, err := recordOf(n) if err != nil { return nil, fmt.Errorf("fejkdata: %s %w", path, err) } - return renderRecord(f.rand, t), nil + return renderRecord(f.rand, t, columns), nil } // RecordTemplate is an inline record compiled, referenced and validated once, // ready to render many times with [RecordTemplate.Fake]. type RecordTemplate struct { - g *Generator - t *template + g *Generator + t *template + columns []string } // Fake renders the record with one draw. func (t *RecordTemplate) Fake() *Record { t.g.mu.Lock() defer t.g.mu.Unlock() - return renderRecord(t.g.rand, t.t) + return renderRecord(t.g.rand, t.t, t.columns) } // NewRecordTemplate compiles an inline record — a JSON object with a format and @@ -128,11 +129,11 @@ func (f *Generator) NewRecordTemplate(input string) (*RecordTemplate, error) { if err != nil { return nil, err } - rt, err := recordOf(t.n) + tm, columns, err := recordOf(t.n) if err != nil { return nil, fmt.Errorf("fejkdata: an inline record %w", err) } - return &RecordTemplate{g: f, t: rt}, nil + return &RecordTemplate{g: f, t: tm, columns: columns}, nil } // FakeRecord compiles and renders an inline record in one call. @@ -144,32 +145,33 @@ func (f *Generator) FakeRecord(input string) (*Record, error) { return t.Fake(), nil } -// recordOf is the fence both record entry points pass: the node is a -// category-level template, it carries no repeat — which composes the format -// rather than projecting columns — and it offers at least one column. -func recordOf(n node) (*template, error) { +// recordOf is the fence both record entry points pass: the node is a template, it +// carries no repeat — which composes the format rather than projecting columns — +// and it offers at least one column. The columns come back with it, fixed for +// every draw the caller goes on to make. +func recordOf(n node) (*template, []string, error) { t, ok := n.(*template) if !ok { - return nil, errors.New("names a choice, not a template; only a category-level template is a record") + return nil, nil, errors.New("names a choice, not a template; a record is a template whose fields are its columns") } if t.repeat != 1 { - return nil, fmt.Errorf("carries repeat %d, which composes its format into one string; a record projects columns instead — drop the repeat and render the record again for more rows", t.repeat) + return nil, nil, fmt.Errorf("carries repeat %d, which composes its format into one string; a record projects columns instead — drop the repeat and render the record again for more rows", t.repeat) } - if len(recordColumns(t)) == 0 { - return nil, errors.New("has no fields, so no columns") + columns := recordColumns(t) + if len(columns) == 0 { + return nil, nil, errors.New("has no fields, so no columns") } - return t, nil + return t, columns, nil } -// renderRecord projects a template's direct fields as columns, drawn once each, -// in name order. A {/path} binding is a render edge, not a column, so it is -// skipped the same way List and the graph do. The columns share one reference -// scope, so two columns that reference one category read one draw of it. -func renderRecord(s *session, t *template) *Record { +// renderRecord draws each column once, in the name order recordOf fixed. The +// columns share one reference scope, so two columns that reference one category +// read one draw of it. +func renderRecord(s *session, t *template, columns []string) *Record { scope := &draws{variant: map[string]node{}, value: map[string]string{}} - r := &Record{} - for _, name := range recordColumns(t) { - r.columns = append(r.columns, Column{Name: name, Value: render(s, t.fields[name], scope)}) + r := &Record{columns: make([]Column, len(columns))} + for i, name := range columns { + r.columns[i] = Column{Name: name, Value: render(s, t.fields[name], scope)} } return r } -- 2.52.0 From be5dfdf9775afbf8cc96137c2e642eaea076893c Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 4 Sep 2026 09:08:23 +0200 Subject: [PATCH 12/21] Tests: overlapping reference columns rejected, every CSV shape round-tripped, an empty column still a row, and an empty --table refused --- cmd/fejkdata/main_test.go | 1 + record_test.go | 54 +++++++++++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/cmd/fejkdata/main_test.go b/cmd/fejkdata/main_test.go index a97ba54..05921b0 100644 --- a/cmd/fejkdata/main_test.go +++ b/cmd/fejkdata/main_test.go @@ -535,6 +535,7 @@ func TestRunRecordMisuse(t *testing.T) { {[]string{"--format", "json", "--separator", ",", "users"}, "--separator joins text values"}, {[]string{"--table", "t", "users"}, "--table names the INSERT target"}, {[]string{"--format", "json", "--table", "t", "users"}, "--table names the INSERT target"}, + {[]string{"--format", "sql", "--table", "", "users"}, "--table names the INSERT target"}, } { code, out, errb := runOut(c.args...) if code != 2 || out != "" || !strings.Contains(errb, c.want) { diff --git a/record_test.go b/record_test.go index e8226f4..78f1202 100644 --- a/record_test.go +++ b/record_test.go @@ -108,19 +108,57 @@ func TestRecordCSV(t *testing.T) { "note": `{"format": "{word}, {word}", "word": ["a", "b,c", "d\"e", "f\n"]}`, }) f := newGenerator(t, dir, WithSeed(1)) - r, err := f.Record("note") + seen := map[string]bool{} + for i := 0; i < 200 && len(seen) < 4; i++ { + r, err := f.Record("note") + if err != nil { + t.Fatal(err) + } + if got := r.CSVHeader(); got != "word" { + t.Fatalf("CSVHeader() = %q, want word", got) + } + rec, err := csv.NewReader(strings.NewReader(r.CSVLine() + "\n")).Read() + if err != nil { + t.Fatalf("CSVLine() is not valid CSV: %v\n%q", err, r.CSVLine()) + } + want := r.Columns()[0].Value + if len(rec) != 1 || rec[0] != want { + t.Fatalf("CSVLine() = %v, want the field value %q round-tripped", rec, want) + } + seen[want] = true + } + if len(seen) != 4 { + t.Fatalf("round-tripped %d of the 4 values; the comma, quote and newline shapes must each survive", len(seen)) + } +} + +func TestRecordCSVEmptyValueStaysARow(t *testing.T) { + dir := writeData(t, map[string]string{"blank": `{"format": "", "note": ""}`}) + f := newGenerator(t, dir, WithSeed(1)) + r, err := f.Record("blank") if err != nil { t.Fatal(err) } - if got := r.CSVHeader(); got != "word" { - t.Errorf("CSVHeader() = %q, want word", got) - } - rec, err := csv.NewReader(strings.NewReader(r.CSVLine() + "\n")).Read() + rows, err := csv.NewReader(strings.NewReader(r.CSVHeader() + "\n" + r.CSVLine() + "\n")).ReadAll() if err != nil { - t.Fatalf("CSVLine() is not valid CSV: %v\n%q", err, r.CSVLine()) + t.Fatalf("csv: %v", err) } - if len(rec) != 1 || rec[0] != r.Columns()[0].Value { - t.Fatalf("CSVLine() = %v, want the field value round-tripped", rec) + if len(rows) != 2 || len(rows[1]) != 1 || rows[1][0] != "" { + t.Fatalf("one empty column parsed to %v, want a header and one row of one empty field", rows) + } +} + +func TestRecordRejectsOverlappingReferenceColumns(t *testing.T) { + dir := writeData(t, map[string]string{ + "cat": `{"format":"","a":[{"format":"A={b}","b":"1"},{"format":"A={b}","b":"2"}]}`, + "row": `{"format":"","whole":"{/cat.a}","inner":"{/cat.a.b}"}`, + }) + f := newGenerator(t, dir, WithSeed(1)) + if _, err := f.Record("row"); err == nil || !strings.Contains(err.Error(), "reads a path into") { + t.Fatalf("Record over an overlapping reference pair = %v, want it rejected the way one format is", err) + } + if _, err := f.Fake("row"); err != nil { + t.Errorf("Fake(row) = %v, want the string view untouched", err) } } -- 2.52.0 From c612c9e73475d7be13ffc133b2be5bf8c0880c61 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 4 Sep 2026 09:10:03 +0200 Subject: [PATCH 13/21] Refuse overlapping reference reads across a record's columns, keep an empty column a CSV row, and reject an empty --table --- README.md | 10 ++++-- cmd/fejkdata/main.go | 7 ++-- record.go | 80 +++++++++++++++++++++++++++++++++++++++----- 3 files changed, 83 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index cb920a3..bc2a91d 100644 --- a/README.md +++ b/README.md @@ -125,9 +125,13 @@ required key — so `"format": ""` carries the fields with an inert format: it renders nothing by `Fake`, and is compiled only so the tree's fences still run. The columns are the point, and their facts stay together: two columns that reference one category — `{/currency.code}` and `{/currency.symbol}` — share one -draw of it, so the record is internally consistent. A field hold, transform or -operand ties fields together within one column as always (see -[Correlated fields](#correlated-fields) and [Decisions](#decisions)). +draw of it, so the record is internally consistent. That one draw is also why two +columns may not read overlapping reference paths — `{/cat.a}` beside `{/cat.a.b}` +is refused, naming the fields to write instead, exactly as +[One draw, one spelling](#one-draw-one-spelling) refuses the pair inside a single +format. A field hold, transform or operand ties fields together within one column +as always (see [Correlated fields](#correlated-fields) and +[Decisions](#decisions)). ## Library diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index 17531f1..6b1ad59 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -216,9 +216,7 @@ type recordFormat struct { line func(r *fejkdata.Record, table string) string } -// recordFormats is every --format that writes records. Adding one is this entry -// alone: the flag check reads the same table the writer dispatches through, so a -// format cannot be accepted and then not written. +// recordFormats is every --format that writes records. var recordFormats = map[string]recordFormat{ "csv": {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }}, "json": {line: func(r *fejkdata.Record, _ string) string { return r.JSON() }}, @@ -245,6 +243,9 @@ func (in invocation) checkFlags() error { if _, ok := recordFormats[in.format]; !ok && in.format != "text" { return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format) } + if in.tableSet && in.table == "" { + return errors.New("--table names the INSERT target, so it cannot be empty") + } if in.tableSet && in.format != "sql" { return errors.New("--table names the INSERT target, so it needs --format sql") } diff --git a/record.go b/record.go index f4ce75a..583b358 100644 --- a/record.go +++ b/record.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "sort" "strings" ) @@ -15,9 +16,8 @@ type Column struct { } // Record is one record rendered from a template: every direct field is a column, -// drawn independently and listed in name order. The template's format is the -// string a [Generator.Fake] call renders; a record is its inverse — each field -// projected as a column instead of composed. +// listed in name order. Each column is its own expansion, so a sibling field is +// local to it, while a reference is drawn once for the whole record. type Record struct { columns []Column } @@ -68,7 +68,11 @@ func csvLine(cols []string) string { w := csv.NewWriter(&b) _ = w.Write(cols) w.Flush() - return strings.TrimSuffix(b.String(), "\n") + line := strings.TrimSuffix(b.String(), "\n") + if line == "" { + return `""` // a blank line is a row every CSV reader drops + } + return line } // SQLInsert renders the record as one INSERT statement into table: identifiers in @@ -145,10 +149,8 @@ func (f *Generator) FakeRecord(input string) (*Record, error) { return t.Fake(), nil } -// recordOf is the fence both record entry points pass: the node is a template, it -// carries no repeat — which composes the format rather than projecting columns — -// and it offers at least one column. The columns come back with it, fixed for -// every draw the caller goes on to make. +// recordOf is the fence both record entry points pass. The columns come back with +// the template, fixed for every draw the caller goes on to make. func recordOf(n node) (*template, []string, error) { t, ok := n.(*template) if !ok { @@ -161,9 +163,71 @@ func recordOf(n node) (*template, []string, error) { if len(columns) == 0 { return nil, nil, errors.New("has no fields, so no columns") } + if err := checkColumnRefs(t, columns); err != nil { + return nil, nil, err + } return t, columns, nil } +// checkColumnRefs rejects two reference reads that overlap across a record's +// columns: one renders a level the other reads a path into, and the record's one +// draw of that level cannot answer for both. checkNoOverlap settles the pair +// within a single format; the record's shared scope is what carries it across +// columns, so the same pair is settled here. A bare reference holds nothing, so it +// is not collected and keeps drawing on its own. +func checkColumnRefs(t *template, columns []string) error { + reads := columnRefs(t, columns) + sort.Slice(reads, func(i, j int) bool { + if reads[i].a.path != reads[j].a.path { + return reads[i].a.path < reads[j].a.path + } + return reads[i].column < reads[j].column + }) + for i, level := range reads { + for _, into := range reads[i+1:] { + if strings.HasPrefix(into.a.path, level.a.path+".") { + return fmt.Errorf("column %q renders {%s}, a level column %q reads a path into with {%s}; name the fields you want instead", + level.column, level.a.name, into.column, into.a.name) + } + } + } + return nil +} + +// columnRef is one held reference read, and the column whose render reaches it. +type columnRef struct { + column string + a arm +} + +// columnRefs lists every held reference read anywhere a column renders, following +// the same edges expand does. +func columnRefs(t *template, columns []string) []columnRef { + var out []columnRef + for _, name := range columns { + seen := map[node]bool{} + var walk func(n node) + walk = func(n node) { + if n == nil || seen[n] { + return + } + seen[n] = true + if tm, ok := n.(*template); ok { + for _, r := range boundReaders(tm.format, tm.bound, tm.refs) { + if a := splitArm(r.name, tm.refs); isRef(a.key) && len(a.tail) > 0 { + out = append(out, columnRef{name, a}) + } + } + } + for _, e := range renderEdges(n) { + walk(e.to) + } + } + walk(t.fields[name]) + } + return out +} + // renderRecord draws each column once, in the name order recordOf fixed. The // columns share one reference scope, so two columns that reference one category // read one draw of it. -- 2.52.0 From 78572ccbbceb44bb7b4e15c7407a02955f41ceb2 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 4 Sep 2026 09:27:52 +0200 Subject: [PATCH 14/21] Tests: a self-referencing column, a bare reference as an operand, and every route the overlap fence must reach --- record_test.go | 65 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/record_test.go b/record_test.go index 78f1202..ae0e6a9 100644 --- a/record_test.go +++ b/record_test.go @@ -149,16 +149,68 @@ func TestRecordCSVEmptyValueStaysARow(t *testing.T) { } func TestRecordRejectsOverlappingReferenceColumns(t *testing.T) { + cat := `{"format":"","a":[{"format":"A={b}","b":"1"},{"format":"A={b}","b":"2"}]}` + for _, c := range []struct{ name, row string }{ + {"sibling columns", `{"format":"","whole":"{/cat.a}","inner":"{/cat.a.b}"}`}, + {"through a nested template", `{"format":"","whole":"{/cat.a}","inner":{"format":"{/cat.a.b}"}}`}, + {"through a column repeat", `{"format":"","whole":"{/cat.a}","inner":{"format":"{/cat.a.b}","repeat":2,"separator":"-"}}`}, + {"through a choice variant", `{"format":"","whole":"{/cat.a}","inner":[{"format":"{/cat.a.b}"},{"format":"{/cat.a.b}!"}]}`}, + {"as a builtin operand", `{"format":"","whole":"{uppercase(/cat.a)}","inner":"{/cat.a.b}"}`}, + } { + f := newGenerator(t, writeData(t, map[string]string{"cat": cat, "row": c.row}), WithSeed(1)) + _, err := f.Record("row") + if err == nil || !strings.Contains(err.Error(), "reads a path into") { + t.Errorf("%s: Record = %v, want the overlap rejected the way one format is", c.name, err) + continue + } + if !strings.Contains(err.Error(), `"whole"`) || !strings.Contains(err.Error(), `"inner"`) { + t.Errorf("%s: error %q names neither column; it must name both", c.name, err) + } + if _, err := f.Fake("row"); err != nil { + t.Errorf("%s: Fake(row) = %v, want the string view untouched", c.name, err) + } + } +} + +func TestRecordRejectsAColumnReadingItsOwnRecord(t *testing.T) { dir := writeData(t, map[string]string{ - "cat": `{"format":"","a":[{"format":"A={b}","b":"1"},{"format":"A={b}","b":"2"}]}`, - "row": `{"format":"","whole":"{/cat.a}","inner":"{/cat.a.b}"}`, + "person": `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"],"full":"{/person.first} {/person.last}"}`, }) f := newGenerator(t, dir, WithSeed(1)) - if _, err := f.Record("row"); err == nil || !strings.Contains(err.Error(), "reads a path into") { - t.Fatalf("Record over an overlapping reference pair = %v, want it rejected the way one format is", err) + if _, err := f.Record("person"); err == nil || !strings.Contains(err.Error(), "points back at this record") { + t.Fatalf("a column referencing its own record = %v, want it refused; it would contradict the columns it reads", err) } - if _, err := f.Fake("row"); err != nil { - t.Errorf("Fake(row) = %v, want the string view untouched", err) +} + +func TestRecordBareReferenceStaysIndependentAsAnOperand(t *testing.T) { + dir := writeData(t, map[string]string{ + "cur": `[{"format":"{code}","code":"aud"},{"format":"{code}","code":"eur"}]`, + "row": `{"format":"","up":"{uppercase(/cur)}","low":"{lowercase(/cur)}"}`, + }) + f := newGenerator(t, dir, WithSeed(1)) + sawMismatch := false + for i := 0; i < 200 && !sawMismatch; i++ { + r, err := f.Record("row") + if err != nil { + t.Fatal(err) + } + m := map[string]string{} + for _, c := range r.Columns() { + m[c.Name] = c.Value + } + sawMismatch = !strings.EqualFold(m["up"], m["low"]) + } + if !sawMismatch { + t.Error("two bare-reference operand columns never disagreed; a bare reference draws on its own, as the plain spelling does") + } + for i := 0; i < 50; i++ { + v, err := f.FakeTemplate("{/cur}|{uppercase(/cur)}") + if err != nil { + t.Fatal(err) + } + if parts := strings.Split(v, "|"); !strings.EqualFold(parts[0], parts[1]) { + t.Fatalf("one format rendered %q; within an expansion a bare reference is still one draw", v) + } } } @@ -309,6 +361,7 @@ func TestInlineRecordErrors(t *testing.T) { {`"hello"`, "has no fields, so no columns"}, {`["a","b"]`, "a record is a template whose fields are its columns"}, {`{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`, "carries repeat 3"}, + {`{"format":"","whole":"{/cur.code}","inner":"{/cur.code.x}"}`, "reads a path into"}, } { if _, err := f.FakeRecord(c.input); err == nil || !strings.Contains(err.Error(), c.want) { t.Errorf("FakeRecord(%q) = %v, want an error naming %q", c.input, err, c.want) -- 2.52.0 From 79827a66d619e624d60c884c39030d6e6e4a918d Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 4 Sep 2026 09:32:29 +0200 Subject: [PATCH 15/21] Fence a record once per node, refuse a column reading its own record, and keep a bare reference independent as an operand --- README.md | 14 +++++---- fejkdata.go | 1 + hold.go | 6 ++-- perf_test.go | 19 +++++++++++++ record.go | 77 ++++++++++++++++++++++++++++++++++++-------------- record_test.go | 16 +++++++++-- 6 files changed, 100 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index bc2a91d..4cc4366 100644 --- a/README.md +++ b/README.md @@ -126,12 +126,14 @@ renders nothing by `Fake`, and is compiled only so the tree's fences still run. The columns are the point, and their facts stay together: two columns that reference one category — `{/currency.code}` and `{/currency.symbol}` — share one draw of it, so the record is internally consistent. That one draw is also why two -columns may not read overlapping reference paths — `{/cat.a}` beside `{/cat.a.b}` -is refused, naming the fields to write instead, exactly as -[One draw, one spelling](#one-draw-one-spelling) refuses the pair inside a single -format. A field hold, transform or operand ties fields together within one column -as always (see [Correlated fields](#correlated-fields) and -[Decisions](#decisions)). +columns may not read overlapping reference *paths* — `{/cat.a}` beside +`{/cat.a.b}` is refused, naming the fields to write instead, as +[One draw, one spelling](#one-draw-one-spelling) refuses that pair inside a single +format. A column may not reference the record it belongs to either: `{/users.first}` +inside `users` would contradict the `first` column beside it, so put a value two +columns share in its own category and reference that. A field hold, transform or +operand ties fields together within one column as always (see +[Correlated fields](#correlated-fields) and [Decisions](#decisions)). ## Library diff --git a/fejkdata.go b/fejkdata.go index 4ede2f1..c558d9b 100644 --- a/fejkdata.go +++ b/fejkdata.go @@ -45,6 +45,7 @@ type Generator struct { mu sync.Mutex rand *session categories map[string]node + records map[node]recordShape } // session is one generator's mutable render state: the seeded rng plus the {seq()} diff --git a/hold.go b/hold.go index 0c7d6b5..05099df 100644 --- a/hold.go +++ b/hold.go @@ -274,10 +274,10 @@ func readField(s *session, t *template, held, refScope *draws, a arm) string { } return render(s, t.fields[a.key], refScope) } - // A reference reads the caller's scope, so its draw outlives this expansion; a - // sibling stays local to it. + // A reference that reads a path reads the caller's scope, so its draw outlives + // this expansion; a sibling, and a reference read whole, stay local to it. d := held - if isRef(a.key) && refScope != nil { + if isRef(a.key) && refScope != nil && len(a.tail) > 0 { d = refScope } if v, read := d.value[a.path]; read { diff --git a/perf_test.go b/perf_test.go index 765525a..8536cac 100644 --- a/perf_test.go +++ b/perf_test.go @@ -51,6 +51,25 @@ func TestNoRenderAllocRegression(t *testing.T) { } } +// A record's fences read the compiled tree, so they belong to New, not to a draw. +// A per-draw walk costs allocations in proportion to the tree; this pins that the +// count does not move with the column count. +func TestNoRecordAllocRegression(t *testing.T) { + for _, s := range []struct{ name, json string }{ + {"record 3 columns", `{"format":"","a":"x","b":"y","c":"z"}`}, + {"record 50 columns", wideTokenJSON(50)}, + } { + f, err := New(WithoutShippedData(), WithDataFS(fstest.MapFS{"x.json": {Data: []byte(s.json)}})) + if err != nil { + t.Fatalf("New(%s): %v", s.name, err) + } + const base = 4.0 + if allocs := testing.AllocsPerRun(10000, func() { f.Record("x") }); allocs > base*1.10 { + t.Errorf("%s: %.1f allocs/op regressed past %.1f (baseline %.1f + 10%%); a record fence running per draw is the usual cause", s.name, allocs, base*1.10, base) + } + } +} + func BenchmarkNestedDepth25(b *testing.B) { benchPath(b, tmpData(b, "deep", nestedJSON(25)), "deep") } func BenchmarkNestedDepth100(b *testing.B) { benchPath(b, tmpData(b, "deep", nestedJSON(100)), "deep") } func BenchmarkWideTokens100(b *testing.B) { diff --git a/record.go b/record.go index 583b358..2798901 100644 --- a/record.go +++ b/record.go @@ -17,7 +17,8 @@ type Column struct { // Record is one record rendered from a template: every direct field is a column, // listed in name order. Each column is its own expansion, so a sibling field is -// local to it, while a reference is drawn once for the whole record. +// local to it, while a reference that reads a path is drawn once for the whole +// record. type Record struct { columns []Column } @@ -104,11 +105,35 @@ func (f *Generator) Record(path string) (*Record, error) { if len(tail) > 0 { return nil, fmt.Errorf("fejkdata: %s descends into %q, a field; only a category-level template is a record", path, tail[0]) } - t, columns, err := recordOf(n) - if err != nil { - return nil, fmt.Errorf("fejkdata: %s %w", path, err) + shape := f.recordShapeOf(n) + if shape.err != nil { + return nil, fmt.Errorf("fejkdata: %s %w", path, shape.err) } - return renderRecord(f.rand, t, columns), nil + return renderRecord(f.rand, shape.t, shape.columns), nil +} + +// recordShape is what recordOf settled about a node: the template to project, its +// columns, or why it is not a record. +type recordShape struct { + t *template + columns []string + err error +} + +// recordShapeOf fences a node once and remembers the answer. The fences read the +// compiled tree, which New fixed, so a repeated Record call on one path pays them +// once rather than per draw. Callers hold the generator's lock. +func (f *Generator) recordShapeOf(n node) recordShape { + if shape, done := f.records[n]; done { + return shape + } + t, columns, err := recordOf(n) + shape := recordShape{t: t, columns: columns, err: err} + if f.records == nil { + f.records = map[node]recordShape{} + } + f.records[n] = shape + return shape } // RecordTemplate is an inline record compiled, referenced and validated once, @@ -169,14 +194,14 @@ func recordOf(n node) (*template, []string, error) { return t, columns, nil } -// checkColumnRefs rejects two reference reads that overlap across a record's -// columns: one renders a level the other reads a path into, and the record's one -// draw of that level cannot answer for both. checkNoOverlap settles the pair -// within a single format; the record's shared scope is what carries it across -// columns, so the same pair is settled here. A bare reference holds nothing, so it -// is not collected and keeps drawing on its own. +// checkColumnRefs rejects the reference reads a record's shared draw cannot answer +// for: one column rendering a level another reads a path into, and a column +// reading the record back through its own path. func checkColumnRefs(t *template, columns []string) error { - reads := columnRefs(t, columns) + reads, err := columnRefs(t, columns) + if err != nil { + return err + } sort.Slice(reads, func(i, j int) bool { if reads[i].a.path != reads[j].a.path { return reads[i].a.path < reads[j].a.path @@ -200,21 +225,32 @@ type columnRef struct { a arm } -// columnRefs lists every held reference read anywhere a column renders, following -// the same edges expand does. -func columnRefs(t *template, columns []string) []columnRef { +// columnRefs lists every reference read that reads a path, anywhere a column +// renders, following the same edges expand does. A read that lands back on the +// record itself names a sibling column, which no draw of the record can answer +// for, so it is reported here rather than collected. +func columnRefs(t *template, columns []string) ([]columnRef, error) { var out []columnRef + var err error for _, name := range columns { seen := map[node]bool{} var walk func(n node) walk = func(n node) { - if n == nil || seen[n] { + if n == nil || seen[n] || err != nil { return } seen[n] = true if tm, ok := n.(*template); ok { for _, r := range boundReaders(tm.format, tm.bound, tm.refs) { - if a := splitArm(r.name, tm.refs); isRef(a.key) && len(a.tail) > 0 { + a := splitArm(r.name, tm.refs) + if !isRef(a.key) { + continue + } + if tm.fields[a.key] == node(t) { + err = fmt.Errorf("column %q reads {%s}, which points back at this record; a column cannot read another column — move the shared value into its own category and reference that", name, a.name) + return + } + if len(a.tail) > 0 { out = append(out, columnRef{name, a}) } } @@ -225,12 +261,11 @@ func columnRefs(t *template, columns []string) []columnRef { } walk(t.fields[name]) } - return out + return out, err } -// renderRecord draws each column once, in the name order recordOf fixed. The -// columns share one reference scope, so two columns that reference one category -// read one draw of it. +// renderRecord draws each column once, in the name order recordOf fixed, over one +// reference scope shared across them. func renderRecord(s *session, t *template, columns []string) *Record { scope := &draws{variant: map[string]node{}, value: map[string]string{}} r := &Record{columns: make([]Column, len(columns))} diff --git a/record_test.go b/record_test.go index ae0e6a9..0de914e 100644 --- a/record_test.go +++ b/record_test.go @@ -152,9 +152,9 @@ func TestRecordRejectsOverlappingReferenceColumns(t *testing.T) { cat := `{"format":"","a":[{"format":"A={b}","b":"1"},{"format":"A={b}","b":"2"}]}` for _, c := range []struct{ name, row string }{ {"sibling columns", `{"format":"","whole":"{/cat.a}","inner":"{/cat.a.b}"}`}, - {"through a nested template", `{"format":"","whole":"{/cat.a}","inner":{"format":"{/cat.a.b}"}}`}, + {"through a nested template", `{"format":"","whole":"{/cat.a}","inner":{"format":"{/cat.a.b} {x}","x":"1"}}`}, {"through a column repeat", `{"format":"","whole":"{/cat.a}","inner":{"format":"{/cat.a.b}","repeat":2,"separator":"-"}}`}, - {"through a choice variant", `{"format":"","whole":"{/cat.a}","inner":[{"format":"{/cat.a.b}"},{"format":"{/cat.a.b}!"}]}`}, + {"through a choice variant", `{"format":"","whole":"{/cat.a}","inner":[{"format":"{/cat.a.b} {x}","x":"1"},{"format":"{/cat.a.b}! {x}","x":"2"}]}`}, {"as a builtin operand", `{"format":"","whole":"{uppercase(/cat.a)}","inner":"{/cat.a.b}"}`}, } { f := newGenerator(t, writeData(t, map[string]string{"cat": cat, "row": c.row}), WithSeed(1)) @@ -172,6 +172,17 @@ func TestRecordRejectsOverlappingReferenceColumns(t *testing.T) { } } +func TestInlineRecordRejectsOverlappingColumns(t *testing.T) { + dir := writeData(t, map[string]string{ + "cat": `{"format":"","a":[{"format":"A={b}","b":"1"},{"format":"A={b}","b":"2"}]}`, + }) + f := newGenerator(t, dir, WithSeed(1)) + _, err := f.FakeRecord(`{"format":"","whole":"{/cat.a}","inner":"{/cat.a.b}"}`) + if err == nil || !strings.Contains(err.Error(), "reads a path into") { + t.Fatalf("inline record over an overlapping pair = %v, want the inline entry point to refuse it too", err) + } +} + func TestRecordRejectsAColumnReadingItsOwnRecord(t *testing.T) { dir := writeData(t, map[string]string{ "person": `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"],"full":"{/person.first} {/person.last}"}`, @@ -361,7 +372,6 @@ func TestInlineRecordErrors(t *testing.T) { {`"hello"`, "has no fields, so no columns"}, {`["a","b"]`, "a record is a template whose fields are its columns"}, {`{"format":"{a}-","repeat":3,"separator":"|","a":["x","y"]}`, "carries repeat 3"}, - {`{"format":"","whole":"{/cur.code}","inner":"{/cur.code.x}"}`, "reads a path into"}, } { if _, err := f.FakeRecord(c.input); err == nil || !strings.Contains(err.Error(), c.want) { t.Errorf("FakeRecord(%q) = %v, want an error naming %q", c.input, err, c.want) -- 2.52.0 From b99526aeb8884e7f6e23323ac774ebe2e0d0310c Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 4 Sep 2026 09:46:39 +0200 Subject: [PATCH 16/21] Tests: a record read whole or as an operand still points back at itself, and Record under concurrent use --- perf_test.go | 6 ++++-- record_test.go | 16 ++++++++++------ shipped_data_test.go | 4 ++++ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/perf_test.go b/perf_test.go index 8536cac..d3ac0cb 100644 --- a/perf_test.go +++ b/perf_test.go @@ -52,8 +52,7 @@ func TestNoRenderAllocRegression(t *testing.T) { } // A record's fences read the compiled tree, so they belong to New, not to a draw. -// A per-draw walk costs allocations in proportion to the tree; this pins that the -// count does not move with the column count. +// A per-draw walk costs allocations in proportion to the tree. func TestNoRecordAllocRegression(t *testing.T) { for _, s := range []struct{ name, json string }{ {"record 3 columns", `{"format":"","a":"x","b":"y","c":"z"}`}, @@ -63,6 +62,9 @@ func TestNoRecordAllocRegression(t *testing.T) { if err != nil { t.Fatalf("New(%s): %v", s.name, err) } + if _, err := f.Record("x"); err != nil { + t.Fatalf("Record(%s): %v", s.name, err) // else the gate would measure the error path + } const base = 4.0 if allocs := testing.AllocsPerRun(10000, func() { f.Record("x") }); allocs > base*1.10 { t.Errorf("%s: %.1f allocs/op regressed past %.1f (baseline %.1f + 10%%); a record fence running per draw is the usual cause", s.name, allocs, base*1.10, base) diff --git a/record_test.go b/record_test.go index 0de914e..01c5d20 100644 --- a/record_test.go +++ b/record_test.go @@ -184,12 +184,16 @@ func TestInlineRecordRejectsOverlappingColumns(t *testing.T) { } func TestRecordRejectsAColumnReadingItsOwnRecord(t *testing.T) { - dir := writeData(t, map[string]string{ - "person": `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"],"full":"{/person.first} {/person.last}"}`, - }) - f := newGenerator(t, dir, WithSeed(1)) - if _, err := f.Record("person"); err == nil || !strings.Contains(err.Error(), "points back at this record") { - t.Fatalf("a column referencing its own record = %v, want it refused; it would contradict the columns it reads", err) + for _, c := range []struct{ name, column string }{ + {"a path into itself", `"full":"{/person.first} {/person.last}"`}, + {"the record read whole", `"whole":"{/person}"`}, + {"the record as an operand", `"up":"{uppercase(/person)}"`}, + } { + person := `{"format":"{first} {last}","first":["Ada","Bo"],"last":["Lovelace","Ek"],` + c.column + `}` + f := newGenerator(t, writeData(t, map[string]string{"person": person}), WithSeed(1)) + if _, err := f.Record("person"); err == nil || !strings.Contains(err.Error(), "points back at this record") { + t.Errorf("%s: Record = %v, want it refused; the column would contradict the columns beside it", c.name, err) + } } } diff --git a/shipped_data_test.go b/shipped_data_test.go index fe1409d..374b7a8 100644 --- a/shipped_data_test.go +++ b/shipped_data_test.go @@ -108,6 +108,10 @@ func TestFakeIsSafeForConcurrentUse(t *testing.T) { return } f.List() + if _, err := f.Record("sv_SE.person"); err != nil { + t.Error(err) + return + } tmpl, err := f.NewTemplate("{/sv_SE.person.last}") if err != nil { t.Error(err) -- 2.52.0 From 85a4cd4475e7894329208914579df870bb88ab80 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 4 Sep 2026 09:47:55 +0200 Subject: [PATCH 17/21] Refuse a column pointing back at its record however it is spelled, and qualify the shared-draw claim to path reads --- README.md | 7 ++++--- record.go | 20 ++++++++------------ 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 4cc4366..b9dc814 100644 --- a/README.md +++ b/README.md @@ -123,9 +123,10 @@ string and a backslash as an escape. A record written only to emit columns still needs a `format` — the grammar's one required key — so `"format": ""` carries the fields with an inert format: it renders nothing by `Fake`, and is compiled only so the tree's fences still run. -The columns are the point, and their facts stay together: two columns that -reference one category — `{/currency.code}` and `{/currency.symbol}` — share one -draw of it, so the record is internally consistent. That one draw is also why two +The columns are the point, and their facts stay together: two columns that read a +path into one category — `{/currency.code}` and `{/currency.symbol}` — share one +draw of it, so the record is internally consistent. A bare `{/currency}` names no +field, so it keeps drawing on its own. That one draw is also why two columns may not read overlapping reference *paths* — `{/cat.a}` beside `{/cat.a.b}` is refused, naming the fields to write instead, as [One draw, one spelling](#one-draw-one-spelling) refuses that pair inside a single diff --git a/record.go b/record.go index 2798901..4a70eaf 100644 --- a/record.go +++ b/record.go @@ -120,9 +120,8 @@ type recordShape struct { err error } -// recordShapeOf fences a node once and remembers the answer. The fences read the -// compiled tree, which New fixed, so a repeated Record call on one path pays them -// once rather than per draw. Callers hold the generator's lock. +// recordShapeOf fences a node once and remembers the answer. Callers hold the +// generator's lock. func (f *Generator) recordShapeOf(n node) recordShape { if shape, done := f.records[n]; done { return shape @@ -225,10 +224,10 @@ type columnRef struct { a arm } -// columnRefs lists every reference read that reads a path, anywhere a column -// renders, following the same edges expand does. A read that lands back on the -// record itself names a sibling column, which no draw of the record can answer -// for, so it is reported here rather than collected. +// columnRefs lists every reference that reads a path, anywhere a column renders, +// following the same edges expand does. A reference landing back on the record +// itself is reported rather than collected, whether it reads a path or the record +// whole: either way the column describes a draw other than its neighbours'. func columnRefs(t *template, columns []string) ([]columnRef, error) { var out []columnRef var err error @@ -241,11 +240,8 @@ func columnRefs(t *template, columns []string) ([]columnRef, error) { } seen[n] = true if tm, ok := n.(*template); ok { - for _, r := range boundReaders(tm.format, tm.bound, tm.refs) { - a := splitArm(r.name, tm.refs) - if !isRef(a.key) { - continue - } + for _, ref := range refTokens(tm.format) { + a := splitArm(ref, tm.refs) if tm.fields[a.key] == node(t) { err = fmt.Errorf("column %q reads {%s}, which points back at this record; a column cannot read another column — move the shared value into its own category and reference that", name, a.name) return -- 2.52.0 From 2310cb1e1a60236d1ab32cf5c72e3aa20de801b1 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 4 Sep 2026 09:53:46 +0200 Subject: [PATCH 18/21] Name every spelling the self-reference fence refuses, and cut the doubled comment --- README.md | 6 +++--- perf_test.go | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b9dc814..d9bbbda 100644 --- a/README.md +++ b/README.md @@ -130,9 +130,9 @@ field, so it keeps drawing on its own. That one draw is also why two columns may not read overlapping reference *paths* — `{/cat.a}` beside `{/cat.a.b}` is refused, naming the fields to write instead, as [One draw, one spelling](#one-draw-one-spelling) refuses that pair inside a single -format. A column may not reference the record it belongs to either: `{/users.first}` -inside `users` would contradict the `first` column beside it, so put a value two -columns share in its own category and reference that. A field hold, transform or +format. A column may not reference the record it belongs to by any spelling: `{/users.first}` +or a bare `{/users}` inside `users` describes a draw other than the columns beside +it, so put a value two columns share in its own category and reference that. A field hold, transform or operand ties fields together within one column as always (see [Correlated fields](#correlated-fields) and [Decisions](#decisions)). diff --git a/perf_test.go b/perf_test.go index d3ac0cb..fc5821c 100644 --- a/perf_test.go +++ b/perf_test.go @@ -52,7 +52,6 @@ func TestNoRenderAllocRegression(t *testing.T) { } // A record's fences read the compiled tree, so they belong to New, not to a draw. -// A per-draw walk costs allocations in proportion to the tree. func TestNoRecordAllocRegression(t *testing.T) { for _, s := range []struct{ name, json string }{ {"record 3 columns", `{"format":"","a":"x","b":"y","c":"z"}`}, -- 2.52.0 From 80524a73b75042df20ff554a2e6775c22fcb48fc Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 4 Sep 2026 10:48:28 +0200 Subject: [PATCH 19/21] Tests: json is an array document and ndjson is one object per line --- cmd/fejkdata/main_test.go | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/cmd/fejkdata/main_test.go b/cmd/fejkdata/main_test.go index 05921b0..fc2c485 100644 --- a/cmd/fejkdata/main_test.go +++ b/cmd/fejkdata/main_test.go @@ -463,16 +463,33 @@ func recordDir(t *testing.T) string { } func TestRunRecordJSON(t *testing.T) { - code, out, errb := runOut("--seed", "1", "--format", "json", "--data-path", recordDir(t), "users") + code, out, errb := runOut("--seed", "1", "--format", "json", "--repeat", "2", "--data-path", recordDir(t), "users") if code != 0 { t.Fatalf("run = %d, stderr=%q", code, errb) } - var m map[string]string - if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &m); err != nil { - t.Fatalf("json output is not one valid JSON object: %v\n%q", err, out) + var arr []map[string]string + if err := json.Unmarshal([]byte(out), &arr); err != nil { + t.Fatalf("json output is not one JSON array: %v\n%q", err, out) } - if len(m) != 2 || m["first"] == "" || m["last"] == "" { - t.Fatalf("json output = %q, want first and last columns", out) + if len(arr) != 2 || len(arr[0]) != 2 || arr[0]["first"] == "" || arr[0]["last"] == "" { + t.Fatalf("json output = %q, want an array of two records with first and last columns", out) + } +} + +func TestRunRecordNDJSON(t *testing.T) { + code, out, errb := runOut("--seed", "1", "--format", "ndjson", "--repeat", "2", "--data-path", recordDir(t), "users") + if code != 0 { + t.Fatalf("run = %d, stderr=%q", code, errb) + } + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + if len(lines) != 2 { + t.Fatalf("ndjson output = %d lines %q, want one object per line", len(lines), out) + } + for _, line := range lines { + var m map[string]string + if err := json.Unmarshal([]byte(line), &m); err != nil || len(m) != 2 { + t.Fatalf("ndjson line %q is not one object: %v", line, err) + } } } @@ -520,9 +537,9 @@ func TestRunRecordInlineTemplate(t *testing.T) { if code != 0 { t.Fatalf("inline record = %d, stderr=%q", code, errb) } - var m map[string]string - if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &m); err != nil || m["x"] != "a" && m["x"] != "b" { - t.Fatalf("inline record json = %q, want a column x (err %v)", out, err) + var arr []map[string]string + if err := json.Unmarshal([]byte(out), &arr); err != nil || len(arr) != 1 || (arr[0]["x"] != "a" && arr[0]["x"] != "b") { + t.Fatalf("inline record json = %q, want one record with a column x (err %v)", out, err) } } @@ -531,7 +548,7 @@ func TestRunRecordMisuse(t *testing.T) { args []string want string }{ - {[]string{"--format", "yaml", "users"}, "--format takes csv, json, sql or text"}, + {[]string{"--format", "yaml", "users"}, "--format takes csv, json, ndjson, sql or text"}, {[]string{"--format", "json", "--separator", ",", "users"}, "--separator joins text values"}, {[]string{"--table", "t", "users"}, "--table names the INSERT target"}, {[]string{"--format", "json", "--table", "t", "users"}, "--table names the INSERT target"}, -- 2.52.0 From f75a9be5b44a531197a01df99fd45373b747749a Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Fri, 4 Sep 2026 10:48:28 +0200 Subject: [PATCH 20/21] Frame json as an array and add ndjson --- README.md | 26 +++++------ cmd/fejkdata/main.go | 104 +++++++++++++++++++++++++++++-------------- 2 files changed, 84 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index d9bbbda..6e5c7c0 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ brace, a bracket or a quote, so the two cannot collide (see | `-s`, `--seed N` | reproducible output | | `-n`, `--repeat N` | render the value N times (up to 1048576), each an independent draw, streamed | | `--separator S` | between repeated values (default a newline) | -| `--format F` | `text` (default), `json`, `csv` or `sql` — a record's columns, one record per row | +| `--format F` | `text` (default), `json`, `ndjson`, `csv` or `sql` — a record's columns, one record per row (json frames them as an array) | | `--table T` | the INSERT target for `--format sql` (default: the path's last segment, or `records` for an inline template) | | `--list` | print every path, then exit | | `--version`, `-h`, `--help` | print, then exit | @@ -81,7 +81,7 @@ For structured output a record writes the row for you. ### Records A record is a template seen as columns: its fields are the columns, its `format` -the whole. `--format json|csv|sql` streams one record per row; the library's +the whole. `--format json|ndjson|csv|sql` writes the records; the library's `Record` (below) hands back the columns. Every column is a string — this is the out-of-scope of typed scalars, see [Decisions](#decisions). Save `mydata/users.json`: @@ -95,17 +95,17 @@ out-of-scope of typed scalars, see [Decisions](#decisions). Save ``` ```sh -fejkdata --seed 1 --data-path ./mydata --format json users # {"first":"Bo","last":"Lovelace"} -fejkdata --seed 1 --data-path ./mydata --format csv users # first,last → Bo,Lovelace -fejkdata --seed 1 --data-path ./mydata --format sql users # INSERT INTO "users" ("first", "last") VALUES ('Bo', 'Lovelace'); -fejkdata --seed 1 --data-path ./mydata --format json --repeat 3 users # three objects, one per line +fejkdata --seed 1 --data-path ./mydata --format json users # [{"first":"Bo","last":"Lovelace"}] +fejkdata --seed 1 --data-path ./mydata --format ndjson users # {"first":"Bo","last":"Lovelace"} +fejkdata --seed 1 --data-path ./mydata --format csv users # first,last → Bo,Lovelace +fejkdata --seed 1 --data-path ./mydata --format sql users # INSERT INTO "users" ("first", "last") VALUES ('Bo', 'Lovelace'); fejkdata --seed 1 --data-path ./mydata --format sql --table people users # INSERT into another table ``` -`--repeat` streams that many records — newline-delimited JSON (one object per -line, NDJSON), a CSV row after a header, or an INSERT in SQL. To fold NDJSON into -a single array, `fejkdata … --format json | jq -s .`. Only a category-level -template is a record; a field, choice or folder errors, and so does a `repeat` on +`--repeat` streams that many records — `json` frames them as one array document, +`ndjson` writes one object per line, `csv` a row after a header, `sql` one INSERT +per line. Only a category-level template is a record; a field, choice or folder +errors, and so does a `repeat` on the template itself, which composes the format into one string rather than projecting columns — ask for more records with `--repeat`. A `repeat` on a column is fine. @@ -165,9 +165,9 @@ r, err = f.FakeRecord(`{"format":"{x}","x":["a","b"]}`) // compile + render inli | `WithoutShippedData()` | load only what you give | A `*Record` carries its columns via `Columns()`, and serializes them with `JSON()` -(one object), `CSVHeader()`/`CSVLine()`, or `SQLInsert(table)` — the same three -shapes the CLI's `--format` streams. `Record` and `FakeRecord` take a record; a -path or template that is not one — a bare string, a choice, or a folder — errors. +(one object), `CSVHeader()`/`CSVLine()`, or `SQLInsert(table)` — the shapes the +CLI's `--format` writes. `Record` and `FakeRecord` take a record; a path or +template that is not one — a bare string, a choice, or a folder — errors. A `*Generator` is safe for concurrent use; a seeded sequence is reproducible only when drawn from one goroutine. Changing how a value is composed shifts the seeded diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index 6b1ad59..cfd5909 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -35,12 +35,12 @@ or a quote). Templates reach the data by reference from the root — argument carrying a bracket, a closing brace or a quote but no valid JSON names neither. -With --format json, csv or sql the argument must name a record — a template whose -fields are its columns — and each render is streamed as one JSON object, one CSV -row (after a header), or one INSERT. +With --format json, ndjson, csv or sql the argument must name a record — a +template whose fields are its columns — and the rows are written as one JSON +array, one JSON object per line, one CSV row (after a header), or one INSERT. -d, --data-path D a data directory to layer over the shipped data (repeatable; last wins on a clash) - --format F output form: text (default), json, csv or sql + --format F output form: text (default), json, ndjson, csv or sql -h, --help print this help, then exit --list list the paths the data offers, then exit --no-shipped-data load only the --data-path directories @@ -209,20 +209,28 @@ func parseArgs(argv []string) (invocation, error) { return in, nil } -// recordFormat is one way to write a record out: the line it renders, and the -// header that precedes the first one, if the format has one. +// recordFormat is one way to write a record out: the line each record renders, +// the header that precedes the first one, and the open/close frame plus the +// between-record separator a document form needs. type recordFormat struct { header func(*fejkdata.Record) string line func(r *fejkdata.Record, table string) string + open string + close string + sep string } -// recordFormats is every --format that writes records. +// recordFormats is every --format that writes records. json frames the records +// as one array document; ndjson is the same column, one object per line. var recordFormats = map[string]recordFormat{ - "csv": {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }}, - "json": {line: func(r *fejkdata.Record, _ string) string { return r.JSON() }}, - "sql": {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }}, + "csv": {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }, sep: "\n"}, + "json": {line: jsonLine, open: "[", close: "]", sep: ",\n"}, + "ndjson": {line: jsonLine, sep: "\n"}, + "sql": {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }, sep: "\n"}, } +func jsonLine(r *fejkdata.Record, _ string) string { return r.JSON() } + // writesRecords reports whether the format writes records rather than plain text. func (in invocation) writesRecords() bool { return in.format != "text" @@ -295,7 +303,10 @@ func (in invocation) options() []fejkdata.Option { // so a render failure comes before anything is written; a write failure surfaces // from Flush, bufio keeping the first one. func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) error { - draw, err := in.draw(f, kind, in.paths[0]) + if in.writesRecords() { + return in.writeRecords(f, kind, w) + } + draw, err := in.textDraw(f, kind, in.paths[0]) if err != nil { return err } @@ -314,43 +325,70 @@ func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) err return out.Flush() } -// draw builds what one render yields: the value's text, or the record's line in -// the chosen format, the header carried ahead of the first one. -func (in invocation) draw(f *fejkdata.Generator, kind argKind, arg string) (func() (string, error), error) { - if !in.writesRecords() { - if kind != argTemplate { - return func() (string, error) { return f.Fake(arg) }, nil - } - t, err := f.NewTemplate(arg) - if err != nil { - return nil, templateError{err} - } - return func() (string, error) { return t.Fake(), nil }, nil +// textDraw builds what one text render yields: the value, from a path or an +// inline template. +func (in invocation) textDraw(f *fejkdata.Generator, kind argKind, arg string) (func() (string, error), error) { + if kind != argTemplate { + return func() (string, error) { return f.Fake(arg) }, nil } + t, err := f.NewTemplate(arg) + if err != nil { + return nil, templateError{err} + } + return func() (string, error) { return t.Fake(), nil }, nil +} + +// recordStream builds the record drawer for the argument, plus the INSERT table +// a sql format names. +func (in invocation) recordStream(f *fejkdata.Generator, kind argKind, arg string) (func() (*fejkdata.Record, error), string, error) { record := func() (*fejkdata.Record, error) { return f.Record(arg) } if kind == argTemplate { t, err := f.NewRecordTemplate(arg) if err != nil { - return nil, templateError{err} + return nil, "", templateError{err} } record = func() (*fejkdata.Record, error) { return t.Fake(), nil } } - format, table, first := recordFormats[in.format], in.table, true + table := in.table if table == "" { table = defaultTable(arg, kind) } - return func() (string, error) { + return record, table, nil +} + +// writeRecords streams a record per line in the chosen format, framing a document +// form with its open/close brackets and a header preceding the first record. +func (in invocation) writeRecords(f *fejkdata.Generator, kind argKind, w io.Writer) error { + record, table, err := in.recordStream(f, kind, in.paths[0]) + if err != nil { + return err + } + format := recordFormats[in.format] + out := bufio.NewWriter(w) + if format.open != "" { + out.WriteString(format.open) + out.WriteByte('\n') + } + for i := 0; i < in.repeat; i++ { r, err := record() if err != nil { - return "", err + return err } - line := format.line(r, table) - if first && format.header != nil { - line = format.header(r) + "\n" + line + if i == 0 && format.header != nil { + out.WriteString(format.header(r)) + out.WriteByte('\n') } - first = false - return line, nil - }, nil + if i > 0 { + out.WriteString(format.sep) + } + out.WriteString(format.line(r, table)) + } + if format.close != "" { + out.WriteByte('\n') + out.WriteString(format.close) + } + out.WriteByte('\n') + return out.Flush() } // defaultTable names the INSERT target when --table is absent: the path's last -- 2.52.0 From b223fcd717d964c177c0d108904cb1c584e2ede5 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sat, 5 Sep 2026 15:29:37 +0200 Subject: [PATCH 21/21] Move deferred and out-of-scope items to todo.md --- AGENTS.md | 15 --------------- README.md | 15 ++------------- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e5ecb93..28cf18d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,18 +8,3 @@ - A standing choice a reader would relitigate goes under Decisions in the README, not in a comment. - A README example is a `json` block that loads and renders as a category; `readme_test.go` runs every one. - Cyclomatic complexity is gated at 14: the table-shaped dispatches (`eachToken`, `calc.factor`, `walkPath`) sit at 13–14 and stay whole; anything else that reaches 14 is decomposed. - -# Deferred - -- **A record cannot ask for an independent reference draw (2026-09-04).** Within - one record every tailed reference to a category is one draw, and a bare - `{/cat}` renders the whole category, so a column wanting its own draw of - `{/cat.field}` has no spelling for it. Left until a use case names which columns - should disagree; premise: a record is one coherent row, which is what columns - are for. Not raised in review before that. - -- **Shipped-data de-duplication (2026-09-02).** `email.json`'s `local` is a - drifted copy of `username.json`, and no shipped file yet uses a held path, an - operand or a reference. Fixed in the data fill before the first tag, when the - shipped set is rewritten anyway; premise: nothing depends on the shipped data's - shape until then. Not raised in review before that. diff --git a/README.md b/README.md index 6e5c7c0..95f444d 100644 --- a/README.md +++ b/README.md @@ -82,8 +82,8 @@ For structured output a record writes the row for you. A record is a template seen as columns: its fields are the columns, its `format` the whole. `--format json|ndjson|csv|sql` writes the records; the library's -`Record` (below) hands back the columns. Every column is a string — this is the -out-of-scope of typed scalars, see [Decisions](#decisions). Save +`Record` (below) hands back the columns. Every column is a string — typed scalars +are on the release checklist, see [`todo.md`](todo.md). Save `mydata/users.json`: ```json @@ -532,12 +532,6 @@ tokens add cost in proportion to the output. field `Fake` renders by dotted path. The `format` is inert to a record — a record-only template writes `"format": ""` — but it is compiled and fenced, so a template that loads renders as whichever shape is asked for. -- **Records emit text; typed scalars are out.** Every value fejkdata yields is a - string, so JSON, CSV and SQL each quote a column as text (`"42"`, `'42'`) rather - than guess a number or a boolean. Emitting unquoted numbers or booleans would - need a per-column `kind`, a parallel scalar system in a format whose promise is - "text means what it says". A column's check digit, number or id is still - valid-by-construction through a builtin; it is serialized as text. - **A record shares one reference draw per category.** Two columns that reference one category — `{/currency.code}` beside `{/currency.symbol}` — read one draw of it, so a record's facts agree the way a template's [correlated @@ -563,11 +557,6 @@ tokens add cost in proportion to the output. row — would vary per draw. A fixed column set is what the CSV and `INSERT` contracts rest on, so the restriction holds even where a particular choice would happen to agree. -- **Filling a Go struct is out of scope.** `Record.Columns()` returns the columns a - caller maps onto a struct themselves, casting each string to the field's type. - gofakeit's `fake:"{firstname}"` tags reflect over an arbitrary struct type and - cast into its fields — a different concern from "data lives in JSON", and one - whose typed casting fejkdata leaves to the caller rather than owning. - **The performance gate asserts allocations, not wall-clock time.** `AllocsPerRun` is deterministic across machines, so a ±10% ceiling does not flake under CI load, while time varies with the machine and its neighbours. A rendering slowdown -- 2.52.0