diff --git a/README.md b/README.md index ab1d65d..594d035 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,8 @@ brace, a bracket or a quote, so the two cannot collide (see | `-s`, `--seed N` | reproducible output | | `-n`, `--repeat N` | render the value N times (up to 1048576), each an independent draw, streamed | | `--separator S` | between repeated values (default a newline) | +| `--format F` | `text` (default), `json`, `csv` or `sql` — a record's columns, one per line | +| `--table T` | the INSERT target for `--format sql` (default: the path's last segment) | | `--list` | print every path, then exit | | `--version`, `-h`, `--help` | print, then exit | @@ -73,6 +75,36 @@ fejkdata --seed 1 --data-path ./mydata sql fejkdata --repeat 100 --data-path ./mydata sql > seed.sql ``` +That `format` string is the free-form spelling — you hand-write the whole row. +For structured output a record writes the row for you. + +### Records + +A record is a template seen as columns: its fields are the columns, its `format` +the whole. `--format json|csv|sql` streams a record per line; the library's +`Record` (below) hands back the columns. Save `mydata/users.json`: + +```json +{ + "format": "{first} {last}", + "first": ["Ada", "Bo"], + "last": ["Lovelace", "Ek"] +} +``` + +```sh +fejkdata --format json users # {"first":"Ada","last":"Lovelace"} +fejkdata --format csv users # first,last → Ada,Lovelace +fejkdata --format sql users # INSERT INTO users (first, last) VALUES ('Ada', 'Lovelace'); +fejkdata --format json --repeat 3 users # three objects, one per line +fejkdata --format sql --table people users # INSERT into another table +``` + +`--repeat` streams that many records — a JSON object per line, a CSV row per +line after a header, an INSERT per line in SQL. Columns render independently, each +a fresh draw, in name order. A column is a string, and each format quotes it as +such; see [Decisions](#decisions) for the typed-scalar and struct-filling scope. + ## Library ```sh @@ -89,6 +121,9 @@ paths := f.List() // every path Fake accepts, sorted v, err = f.FakeTemplate("name: {/sv_SE.person.last}") // compile + render in one call t, err := f.NewTemplate(`{"format":"name: {x}","x":["bosse","lina"]}`) // compile once v = t.Fake() // render many times, no re-parse +r, err := f.Record("users") // one record: each field a column +s := r.JSON() // {"first":"Ada","last":"Lovelace"} +r, err = f.FakeRecord(`{"format":"{x}","x":["a","b"]}`) // compile + render inline ``` | Option | | @@ -98,6 +133,11 @@ v = t.Fake() // render many times, | `WithDataFS(fsys)` | layer an `fs.FS`, such as your own `embed.FS` | | `WithoutShippedData()` | load only what you give | +A `*Record` carries its columns via `Fields()`, and serializes them with `JSON()` +(one object), `CSVHeader()`/`CSVLine()`, or `SQLInsert(table)` — the same three +shapes the CLI's `--format` streams. `Record` and `FakeRecord` take a record; a +path or template that is not one — a bare string, a choice, or a folder — errors. + A `*Generator` is safe for concurrent use; a seeded sequence is reproducible only when drawn from one goroutine. Changing how a value is composed shifts the seeded stream for that value and everything drawn after it. @@ -454,6 +494,22 @@ tokens add cost in proportion to the output. - **Samples say what they emit, transforms what they do.** `{upper(2)}` is two letters, `{uppercase(x)}` is `x` upper-cased; one name for both would turn on whether the argument looks like a number. +- **A record is a template seen as columns, not a second schema format.** A + template's `format` composes its fields into one string; `Record` and + `--format` project the same fields as columns. Two views of one dataset, so a + record author writes the same JSON they already know, and a column is the same + field `Fake` renders by dotted path. +- **Records emit text; typed scalars are out.** Every value fejkdata yields is a + string, so JSON, CSV and SQL each quote a column as text (`"42"`, `'42'`) rather + than guess a number or a boolean. Emitting unquoted numbers or booleans would + need a per-column `kind`, a parallel scalar system in a format whose promise is + "text means what it says". A column's check digit, number or id is still + valid-by-construction through a builtin; it is serialized as text. +- **Filling a Go struct is out of scope.** `Record.Fields()` returns the columns a + caller maps onto a struct themselves. gofakeit's `fake:"{firstname}"` tags + reflect over an arbitrary struct type and cast into its fields — a different + concern from "data lives in JSON", and one a JSON record feeds without fejkdata + owning the reflection. - **The performance gate asserts allocations, not wall-clock time.** `AllocsPerRun` is deterministic across machines, so a ±10% ceiling does not flake under CI load, while time varies with the machine and its neighbours. A rendering slowdown @@ -500,6 +556,7 @@ fejkdata.go Generator, New, options, the embedded data set, List node.go the node model and JSON -> node compilation path.go the dotted-path walk, and proving a path resolves render.go Fake and the recursive renderer (choices, format strings, expansions) +record.go records: Record, the JSON/CSV/SQL serializers, and their entry points inline.go inline templates: Template, NewTemplate, FakeTemplate, and their compile and link template.go the {token} grammar: scanning, tokens, operands, validation, compiling a format hold.go the hold: one draw per expansion for paths and operands, and its fences diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index bf69ed0..3dbd5fb 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -34,13 +34,19 @@ or a quote). Templates reach the data by reference from the root — argument carrying a bracket, a closing brace or a quote but no valid JSON names neither. +With --format json, csv or sql the argument must name a record — a template whose +fields are its columns — and each render is streamed as one JSON object, one CSV +row (after a header), or one INSERT. + -d, --data-path D a data directory to layer over the shipped data (repeatable; last wins on a clash) + --format F output form: text (default), json, csv or sql -h, --help print this help, then exit --list list the paths the data offers, then exit --no-shipped-data load only the --data-path directories -n, --repeat N render the value N times, 1..1048576 (default 1) -s, --seed N seed for reproducible output --separator S string between repeated values (default newline) + --table T the INSERT target for --format sql (default: the path's last segment) --version print the version, then exit Flags may come before or after ; -- ends the flags. A short flag's @@ -49,6 +55,8 @@ value attaches or follows (-n3, -n 3); short flags bundle (-hn 3). type invocation struct { dirs []string + format string + formatSet bool help bool list bool noShipped bool @@ -59,6 +67,8 @@ type invocation struct { seeded bool separator string separatorSet bool + table string + tableSet bool version bool } @@ -71,6 +81,7 @@ type flagDef struct { var flagDefs = []flagDef{ {"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }}, + {"format", "", true, func(in *invocation, v string) error { in.format, in.formatSet = v, true; return nil }}, {"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }}, {"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }}, {"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }}, @@ -91,6 +102,7 @@ var flagDefs = []flagDef{ return nil }}, {"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }}, + {"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }}, {"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }}, } @@ -158,7 +170,7 @@ func splitFlags(arg string) ([]flagArg, error) { } func parseArgs(argv []string) (invocation, error) { - in := invocation{repeat: 1, separator: "\n"} + in := invocation{repeat: 1, separator: "\n", format: "text"} for i := 0; i < len(argv); i++ { arg := argv[i] if arg == "--" { @@ -196,14 +208,41 @@ func parseArgs(argv []string) (invocation, error) { return in, nil } +// recordFormat reports whether the format names a record output (json, csv or +// sql) rather than the default text. +func (in invocation) recordFormat() bool { + return in.formatSet && in.format != "text" +} + +// checkFlags rejects a flag value or combination that cannot run. +func (in invocation) checkFlags() error { + if in.formatSet { + switch in.format { + case "text", "json", "csv", "sql": + default: + return fmt.Errorf("--format takes text, json, csv or sql, got %q", in.format) + } + } + if in.tableSet && in.format != "sql" { + return errors.New("--table names the INSERT target, so it needs --format sql") + } + if in.recordFormat() && in.separatorSet { + return errors.New("--separator joins text values, so it has no effect with --format " + in.format) + } + return nil +} + // check rejects a flag combination or an argument that cannot run, and reports // what the argument names, so its shape is settled before any data is read. func (in invocation) check() (argKind, error) { + if err := in.checkFlags(); err != nil { + return argPath, err + } if in.list && len(in.paths) > 0 { return argPath, errors.New("--list takes no path") } - if in.list && (in.repeatSet || in.separatorSet) { - return argPath, errors.New("--list takes no --repeat or --separator") + if in.list && (in.repeatSet || in.separatorSet || in.formatSet || in.tableSet) { + return argPath, errors.New("--list takes no --repeat, --separator, --format or --table") } if in.list { return argPath, nil @@ -234,6 +273,9 @@ func (in invocation) options() []fejkdata.Option { // from Flush, bufio keeping the first one. func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) error { arg := in.paths[0] + if in.recordFormat() { + return in.writeRecords(f, kind, arg, w) + } draw := func() (string, error) { return f.Fake(arg) } if kind == argTemplate { t, err := f.NewTemplate(arg) @@ -257,6 +299,52 @@ func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) err return out.Flush() } +// writeRecords streams a record per line in the chosen format: one JSON object +// per line, a CSV header then a row per line, or one INSERT per line. +func (in invocation) writeRecords(f *fejkdata.Generator, kind argKind, arg string, w io.Writer) error { + draw := func() (*fejkdata.Record, error) { return f.Record(arg) } + if kind == argTemplate { + t, err := f.NewRecordTemplate(arg) + if err != nil { + return templateError{err} + } + draw = func() (*fejkdata.Record, error) { return t.Fake(), nil } + } + out := bufio.NewWriter(w) + table := in.table + if table == "" { + table = defaultTable(arg, kind) + } + for i := 0; i < in.repeat; i++ { + r, err := draw() + if err != nil { + return err + } + switch in.format { + case "json": + fmt.Fprintln(out, r.JSON()) + case "csv": + if i == 0 { + fmt.Fprintln(out, r.CSVHeader()) + } + fmt.Fprintln(out, r.CSVLine()) + case "sql": + fmt.Fprintln(out, r.SQLInsert(table)) + } + } + return out.Flush() +} + +// defaultTable names the INSERT target when --table is absent: the path's last +// segment, or "records" for an inline template that sits in no folder. +func defaultTable(arg string, kind argKind) string { + if kind == argTemplate { + return "records" + } + segments := strings.Split(arg, ".") + return segments[len(segments)-1] +} + // templateError marks a render failure that is the argument's own fault — an // inline template that does not compile. run reports it as misuse (exit 2, with a // pointer to --help), unlike an unknown path, which is a runtime error (exit 1). diff --git a/record.go b/record.go new file mode 100644 index 0000000..9ef7cbd --- /dev/null +++ b/record.go @@ -0,0 +1,180 @@ +package fejkdata + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "sort" + "strings" +) + +// Field is one rendered column of a record. +type Field struct { + Name string + Value string +} + +// Record is one record rendered from a template: every direct field is a column, +// drawn independently and listed in name order. The template's format is the +// string a [Generator.Fake] call renders; a record is its inverse — each field +// projected as a column instead of composed. +type Record struct { + fields []Field +} + +// Fields returns the record's columns in name order. +func (r *Record) Fields() []Field { + return append([]Field(nil), r.fields...) +} + +// JSON renders the record as one JSON object, every column a string. +func (r *Record) JSON() string { + m := make(map[string]string, len(r.fields)) + for _, f := range r.fields { + m[f.Name] = f.Value + } + b, _ := json.Marshal(m) + return string(b) +} + +// CSVHeader renders the column names as one CSV header line. +func (r *Record) CSVHeader() string { + return csvLine(r.names()) +} + +// CSVLine renders the column values as one CSV row. +func (r *Record) CSVLine() string { + return csvLine(r.values()) +} + +func (r *Record) names() []string { + out := make([]string, len(r.fields)) + for i, f := range r.fields { + out[i] = f.Name + } + return out +} + +func (r *Record) values() []string { + out := make([]string, len(r.fields)) + for i, f := range r.fields { + out[i] = f.Value + } + return out +} + +func csvLine(cols []string) string { + var b strings.Builder + w := csv.NewWriter(&b) + _ = w.Write(cols) + w.Flush() + return strings.TrimSuffix(b.String(), "\n") +} + +// SQLInsert renders the record as one INSERT statement into table, every column a +// single-quoted string literal. +func (r *Record) SQLInsert(table string) string { + cols := make([]string, len(r.fields)) + vals := make([]string, len(r.fields)) + for i, f := range r.fields { + cols[i] = f.Name + vals[i] = "'" + strings.ReplaceAll(f.Value, "'", "''") + "'" + } + return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);", table, strings.Join(cols, ", "), strings.Join(vals, ", ")) +} + +// Record renders a path as one record: the template it names, with each direct +// field drawn as a column. A path naming a folder or a value that is not a +// template — a bare string or a choice, which have no fields — is an error. +func (f *Generator) Record(path string) (*Record, error) { + f.mu.Lock() + defer f.mu.Unlock() + n, err := descend(f.rand, &group{children: f.categories}, strings.Split(path, ".")) + if err != nil { + return nil, fmt.Errorf("fejkdata: %s: %w", path, err) + } + if _, ok := n.(*group); ok { + return nil, fmt.Errorf("fejkdata: %s names a folder, not a value", path) + } + t, ok := n.(*template) + if !ok { + return nil, fmt.Errorf("fejkdata: %s does not name a record; it has no fields to project", path) + } + r := renderRecord(f.rand, t) + if len(r.fields) == 0 { + return nil, fmt.Errorf("fejkdata: %s has no fields, so no columns", path) + } + return r, nil +} + +// RecordTemplate is an inline record compiled, referenced and validated once, +// ready to render many times with [RecordTemplate.Fake]. +type RecordTemplate struct { + g *Generator + n node +} + +// Fake renders the record with one draw. +func (t *RecordTemplate) Fake() *Record { + t.g.mu.Lock() + defer t.g.mu.Unlock() + return renderRecord(t.g.rand, t.n.(*template)) +} + +// NewRecordTemplate compiles an inline record — a JSON object with a format and +// fields — and binds its references against the loaded tree. +func (f *Generator) NewRecordTemplate(input string) (*RecordTemplate, error) { + n, err := compileInput(input) + if err != nil { + return nil, fmt.Errorf("fejkdata: %w", err) + } + t, ok := n.(*template) + if !ok { + return nil, fmt.Errorf("fejkdata: an inline record is a JSON object with a format and fields, not a choice") + } + if len(recordColumns(t)) == 0 { + return nil, fmt.Errorf("fejkdata: an inline record needs at least one field to project as a column") + } + scope := inlineScope(t) + if err := linkNodeRefs(scope, f.categories); err != nil { + return nil, fmt.Errorf("fejkdata: %w", err) + } + if err := checkScope(scope); err != nil { + return nil, fmt.Errorf("fejkdata: %w", err) + } + return &RecordTemplate{g: f, n: t}, nil +} + +// FakeRecord compiles and renders an inline record in one call. +func (f *Generator) FakeRecord(input string) (*Record, error) { + t, err := f.NewRecordTemplate(input) + if err != nil { + return nil, err + } + return t.Fake(), nil +} + +// renderRecord projects a template's direct fields as columns, drawn once each, +// in name order. A {/path} binding is a render edge, not a column, so it is +// skipped the same way List and the graph do. +func renderRecord(s *session, t *template) *Record { + r := &Record{} + for _, name := range recordColumns(t) { + r.fields = append(r.fields, Field{Name: name, Value: render(s, t.fields[name])}) + } + return r +} + +// recordColumns is the sorted non-reference field names — the columns a record +// projects. A {/path} binding is carried in fields under its root path, so only a +// name that is not a reference is a column. +func recordColumns(t *template) []string { + var names []string + for name := range t.fields { + if !isRef(name) { + names = append(names, name) + } + } + sort.Strings(names) + return names +}