diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bffbe7..26c3a37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ replacement, and each removed path, column or flag. fejkdata`, and adds nothing under `--format text`: `echo`'s newline ends the render, and `--separator` defaults to empty. Whitespace around a JSON template is dropped. A lone reference, `{/users}`, is the record `users` under `--format`. An argument other than a flag, an - empty or blank stdin, one holding only `null`, or nothing piped in, exits 2. + empty or blank stdin, or nothing piped in, exits 2. - Table categories: a category JSON naming a `rows` TSV beside it, with the options `key`, `name`, `weight` and `parent`; a path selects a row by key or name, `misc.territory[SE]`, and descends to a linked table by name, one path drawing each diff --git a/README.md b/README.md index dbdb9e4..bb6253a 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,8 @@ these paths; a template reads one as `{/path}`. Under `--format text`, the default, what you send is what renders, and fejkdata adds nothing. A format string keeps every byte, so `echo`, a file or a heredoc ends each render with its newline, and `printf '%s'` sends none; `-n` joins renders with `--separator`, empty by default. A JSON template is -an object, array or string, and the whitespace around it is dropped; `null`, `42` or -`true` alone is refused. A JSON template that should end +any JSON value, and the whitespace around it is dropped; `42` or `true` alone renders its +text, and `null` alone renders nothing. A JSON template that should end each render in a newline writes `\n` in its format, from a quoted heredoc or a file. A lone reference, one `{/…}` and nothing else, is the record it names under `--format`, one newline after it allowed. Some shells' `echo` reads a backslash as an escape, so pipe a @@ -55,17 +55,17 @@ template holding one from a quoted heredoc or a file. The rest of stdin's rules | `--no-shipped-data` | load only the `--data-path` directories | | `-s`, `--seed N` | same seed, version and data: identical output | | `-n`, `--repeat N` | render the template N times (up to 1048576), each an independent draw, streamed | -| `--separator S` | between repeated renders (default empty) | +| `--separator S` | between repeated renders (default empty); a record `--format` ignores it | | `--format F` | `text` (default), `json`, `ndjson`, `csv` or `sql` — a record's columns, one record per row (json frames them as an array) | -| `--table T` | the INSERT target for `--format sql` (default: a lone reference's last segment, such as `users` for `{/users}`, else `records`) | -| `--list` | print every path, then exit | +| `--table T` | the INSERT target for `--format sql`, ignored under any other (default: a lone reference's last segment, such as `users` for `{/users}`, else `records`) | +| `--list` | print every path, then exit; `--repeat`, `--separator`, `--format` and `--table` do nothing here | | `--version`, `-h`, `--help` | print, then exit | `--name value` and `--name=value` both work, a short flag's value attaches or follows (`-n3`, `-n 3`) and short flags bundle (`-hn 3`) — see [Decisions](docs/decisions.md#flags-follow-getopt_long); flags go anywhere, `--` ends them. Exit codes: `0` success, `1` runtime error (missing dir, a lone reference to nothing under `--format`), `2` misuse — a bad flag, an argument -other than a flag, an empty, blank, `null` or unreadable stdin, nothing piped in, or a +other than a flag, an empty, blank or unreadable stdin, nothing piped in, or a template that does not compile. From a checkout: `go run ./cmd/fejkdata …`. @@ -327,8 +327,8 @@ ok, err := fejkdata.IsTemplate(arg) // an inline template by its shape, el | `WithDataFS(fsys)` | layer an `fs.FS`, such as your own `embed.FS` | | `WithoutShippedData()` | load only what you give | -`New` refuses a mistake in the data, and `NewTemplate` one in the template. On a loaded -generator: +`New` refuses data that breaks the grammar, could mean two things or cannot render a valid +value, and `NewTemplate` such a template. On a loaded generator: - `Fake` fails only for a path that names nothing, could name two things, or reads one draw of a level carrying a `repeat`, with the same error every call. @@ -428,9 +428,10 @@ An item of a choice may carry a `weight` (default `1`) to skew its odds: Renders `070-412 38 91` ten times as often as `08-…`. A string item is weighted by writing it as `{ "format": "AB", "weight": 3 }`. A repeated item counts as written, -so `["a", "a", "b"]` draws `a` two times in three. Rejected at load: a weight that is -negative, non-numeric or `0` (never drawn — remove the item), and one other than `1` -where nothing is drawn against it: outside a choice, or on the item of a one-item choice. +so `["a", "a", "b"]` draws `a` two times in three, and an item of weight `0` is never +drawn. A weight where nothing is drawn against it, outside a choice or on the item of a +one-item choice, does nothing. Rejected at load: a weight that is negative or not a +number, and a choice whose every weight is `0`. ### Repeat @@ -441,9 +442,9 @@ that many times — each an independent draw — joined by `separator` (default { "format": "{word}", "repeat": 3, "separator": " ", "word": ["foo", "bar", "baz"] } ``` -Renders e.g. `bar foo baz`. Rejected at load: a `separator` other than `""` without a -`repeat` above `1`, and a `repeat` that multiplies to more than 1 048 576 renders along any path of -nested repeats. +Renders e.g. `bar foo baz`. A `separator` without a `repeat` above `1` does nothing. +Rejected at load: a `repeat` that multiplies to more than 1 048 576 renders along any +path of nested repeats. ### Datatype @@ -459,8 +460,8 @@ writes `42` rather than `"42"` and `sql` a bare literal; a column without one, o ``` Writes e.g. `{"id":1,"paid":true,"total":59.97}`. A column is a field of the top-level -template, or an item of a choice standing in for one; `datatype` anywhere else, bar -`string`, is a load error. A typed column holds one value, alone in its format: a +template, or an item of a choice standing in for one; `datatype` anywhere else does +nothing. A typed column holds one value, alone in its format: a literal, one `{int()}`, `{float()}`, `{seq()}` or `{calc()}` call, or a read that lands only on such values. `integer` is an int64 written `-?(0|[1-9][0-9]*)` — `{float()}` prints one at `0` decimals within int64 — `number` a JSON number, `boolean` `true` or `false`. A value its @@ -497,9 +498,9 @@ renders a null as `""`. The other items' weights skew its odds: { "format": "", "deleted_at": null, "middle": [null, { "format": "{n}", "n": ["Ann", "Eva"], "weight": 3 }] } ``` -`deleted_at` is null every draw, `middle` a name three draws in four. Rejected at -load: `null` anywhere but a column, where it only renders `""`, naming `""`, and a -column whose items hold different datatypes. +`deleted_at` is null every draw, `middle` a name three draws in four. Anywhere but a +column, `null` renders `""`. Rejected at load: a column whose items hold different +datatypes. ### Table @@ -536,8 +537,8 @@ in a cell draws digits and `{/misc.uuid}` reads a reference, while `{name}` in a is refused, since a cell has no sibling. Each cell may select its own row of another table, `{/misc.currency[SEK].symbol}` on one row and `{/misc.currency[EUR].symbol}` on the next, since only one row renders. `New` proves the header, the options and every -cell token, and refuses a TSV no category names, a key that is empty or repeats, a -weight that is not a positive number, and a key or name holding `[`, `]`, `{`, `}`, +cell token, and refuses a key that is empty or repeats, a weight that is negative or not +a number, every weight `0` across the table or inside one parent row, and a key or name holding `[`, `]`, `{`, `}`, `"` or `|`, which a selector cannot spell; `New` maps the keys, and a name or parent column is mapped on the first draw that selects by name or descends through the table. A `name` needs a `key`, since a name naming several rows is reported by their keys, or a `parent`, inside whose row a name names one row, so `first-name[Kim]` @@ -631,11 +632,7 @@ namespace too. A `{name(args)}` token calls a builtin. Arguments are checked at `New`: a bad count, range, country or expression fails fast; an integer may carry a sign or leading -zeros (`+5` and `05` are `5`); bounds are finite; a constant sample — `int` or `float` -with min equal to max, `date` with from equal to to and a layout with no clock field, -and a `date` or `time` layout naming no field — is rejected naming the text to write -instead; -and a length, count or decimal place beyond a sane maximum is rejected, so a +zeros (`+5` and `05` are `5`); bounds are finite; and a length, count or decimal place beyond a sane maximum is rejected, so a fat-fingered `hex(2000000000)` never tries to allocate gigabytes. Every builtin draws only from the seed — a time-based id takes its timestamp from the rng, not the clock — so seeded output stays reproducible. @@ -677,10 +674,10 @@ between the two days is reachable, so a layout with a clock draws the time too, `from` may equal `to`, which is that one day. The instant is UTC, so a zone in the layout prints `UTC` or `Z`. The quotes delimit a layout outside a selector only, so `[O'Fallon]` in an -argument stays a name. Rejected at `New`: a bound that is no calendar date, or not -before the other; an unquoted layout, naming the single-quoted one; a layout naming no field, which is -text, as is one day in a layout with no clock; for `date` a layout naming no date -field, naming `time`; and for `time` a layout naming a date field, naming `date`. `{seq()}` spans `Fake` +argument stays a name. A layout naming no field prints itself, `time` prints a date +field as 1970-01-01, and `date` over one day with no clock field prints that day. +Rejected at `New`: a bound that is no calendar date, or after the other, and an unquoted +or double-quoted layout, naming the single-quoted one. `{seq()}` spans `Fake` calls and `repeat`, resets with a new generator, and is the natural primary key for the SQL example above. @@ -778,7 +775,6 @@ Refused at `New`, each error naming what to write instead: one; - a read of a name inside the field bound to it: read it outside that field; - a binding in a choice's item, which every other item would leave unbound; -- a binding nothing reads; - a path through a name that selects a row; - `{n}` beside `{n.w}` where `n`'s category reads `w` twice, as in `{w}-{w}` or `{w.a} {w.b}`; diff --git a/cmd/fejkdata/main.go b/cmd/fejkdata/main.go index 6f3db37..9e28fb4 100644 --- a/cmd/fejkdata/main.go +++ b/cmd/fejkdata/main.go @@ -45,8 +45,8 @@ path into one ({/sv_SE.person.last}), a table's row by key or name Under --format text, the default, what you send is what renders, and fejkdata adds nothing: a format string keeps every byte, so echo's newline ends each render, and printf '%s' sends none. -n joins renders with --separator, empty by default. A JSON -template is an object, array or string, and the whitespace around it is dropped; null, -42 or true alone is refused. A quoted heredoc, <<'EOF', +template is any JSON value, and the whitespace around it is dropped; 42 or true alone +renders its text, and null nothing. A quoted heredoc, <<'EOF', passes $, a backslash or a quote as written. With --format json, ndjson, csv or sql the template must be a record — a JSON @@ -58,12 +58,14 @@ one INSERT. -d, --data-path D a data directory to layer over the shipped data (repeatable; last wins on a clash) --format F output form: text (default), json, ndjson, csv or sql -h, --help print this help, then exit - --list list the paths the data offers, each written after {/ in a template, then exit + --list list the paths the data offers, each written after {/ in a template, then exit; + --repeat, --separator, --format and --table do nothing here --no-shipped-data load only the --data-path directories -n, --repeat N render the template N times, 1..1048576 (default 1) -s, --seed N same seed, version and data: identical output - --separator S string between repeated renders (default empty) - --table T the INSERT target for --format sql (default: a lone reference's last segment, else records) + --separator S string between repeated renders (default empty); a record --format ignores it + --table T the INSERT target for --format sql, ignored under any other + (default: a lone reference's last segment, else records) --version print the version, then exit A short flag's value attaches or follows (-n3, -n 3); short flags bundle (-hn 3); @@ -71,22 +73,19 @@ A short flag's value attaches or follows (-n3, -n 3); short flags bundle (-hn 3) ` type invocation struct { - args []string - dirs []string - format string - formatSet bool - help bool - list bool - noShipped bool - repeat int - repeatSet bool - seed uint64 - seeded bool - separator string - separatorSet bool - table string - tableSet bool - version bool + args []string + dirs []string + format string + help bool + list bool + noShipped bool + repeat int + seed uint64 + seeded bool + separator string + table string + tableSet bool + version bool } type flagDef struct { @@ -98,7 +97,7 @@ type flagDef struct { var flagDefs = []flagDef{ {"data-path", "d", true, func(in *invocation, v string) error { in.dirs = append(in.dirs, v); return nil }}, - {"format", "", true, func(in *invocation, v string) error { in.format, in.formatSet = v, true; return nil }}, + {"format", "", true, func(in *invocation, v string) error { in.format = v; return nil }}, {"help", "h", false, func(in *invocation, _ string) error { in.help = true; return nil }}, {"list", "", false, func(in *invocation, _ string) error { in.list = true; return nil }}, {"no-shipped-data", "", false, func(in *invocation, _ string) error { in.noShipped = true; return nil }}, @@ -107,7 +106,7 @@ var flagDefs = []flagDef{ if err != nil || n < 1 || n > fejkdata.MaxRepeat { return fmt.Errorf("--repeat needs an integer in 1..%d, got %q", fejkdata.MaxRepeat, v) } - in.repeat, in.repeatSet = n, true + in.repeat = n return nil }}, {"seed", "s", true, func(in *invocation, v string) error { @@ -118,7 +117,7 @@ var flagDefs = []flagDef{ in.seed, in.seeded = n, true return nil }}, - {"separator", "", true, func(in *invocation, v string) error { in.separator, in.separatorSet = v, true; return nil }}, + {"separator", "", true, func(in *invocation, v string) error { in.separator = v; return nil }}, {"table", "", true, func(in *invocation, v string) error { in.table, in.tableSet = v, true; return nil }}, {"version", "", false, func(in *invocation, _ string) error { in.version = true; return nil }}, } @@ -266,15 +265,9 @@ func (in invocation) checkFlags() error { if _, ok := recordFormats[in.format]; !ok && in.format != "text" { return fmt.Errorf("--format takes %s, got %q", formatNames(), in.format) } - if in.tableSet && in.table == "" { + if in.tableSet && in.table == "" && in.format == "sql" { return errors.New("--table names the INSERT target, so it cannot be empty") } - if in.tableSet && in.format != "sql" { - return errors.New("--table names the INSERT target, so it needs --format sql") - } - if in.writesRecords() && in.separatorSet { - return errors.New("--separator joins text values, so it has no effect with --format " + in.format) - } return nil } @@ -314,9 +307,6 @@ func (in invocation) check() error { if len(in.args) > 0 { return operandError(in.args[0]) } - if in.list && (in.repeatSet || in.separatorSet || in.formatSet || in.tableSet) { - return errors.New("--list takes no --repeat, --separator, --format or --table") - } return nil } @@ -459,9 +449,6 @@ func parseInput(raw string) (input, error) { if trimmed == "" { return input{}, fmt.Errorf("stdin holds no template: %s", pipeHint) } - if trimmed == "null" { - return input{}, fmt.Errorf("stdin holds only null, as jq prints for a missing key: %s", pipeHint) - } text := raw if json.Valid([]byte(trimmed)) { text = trimmed diff --git a/datatype.go b/datatype.go index 3fd8b3f..0f94bd8 100644 --- a/datatype.go +++ b/datatype.go @@ -20,7 +20,8 @@ const ( DataTypeBoolean DataType = datatype.Boolean ) -// datatypeOf reads a template's "datatype", nil where it carries none. +// datatypeOf reads a template's "datatype", nil where it carries none or sits outside a column, +// where no datatype applies. func datatypeOf(m map[string]any, pos position) (*DataType, error) { v, ok := m["datatype"] if !ok { @@ -34,8 +35,8 @@ func datatypeOf(m map[string]any, pos position) (*DataType, error) { if name != d.String() { continue } - if pos != inColumn && d != DataTypeString { - return nil, errors.New(`datatype only types a record column — a field of the top-level template — so it has no effect here; drop "datatype"`) + if pos != inColumn { + return nil, nil } return &d, nil } diff --git a/fejkdata_test.go b/fejkdata_test.go index b97391d..1be0b22 100644 --- a/fejkdata_test.go +++ b/fejkdata_test.go @@ -1,13 +1,13 @@ package fejkdata import ( - "encoding/json" "errors" "io/fs" "strings" "testing" "testing/fstest" + "github.com/larvit/fejkdata/internal/datafiles" "github.com/larvit/fejkdata/internal/drawstate" ) @@ -104,8 +104,8 @@ func engine(seed uint64) *Generator { // parse unmarshals a JSON template fragment into its dynamic form. func parse(t *testing.T, s string) any { t.Helper() - var v any - if err := json.Unmarshal([]byte(s), &v); err != nil { + v, err := datafiles.DecodeJSON([]byte(s)) + if err != nil { t.Fatalf("parse %q: %v", s, err) } return v diff --git a/inline.go b/inline.go index 6ee548f..8e79f45 100644 --- a/inline.go +++ b/inline.go @@ -1,9 +1,9 @@ package fejkdata import ( - "encoding/json" "fmt" + "github.com/larvit/fejkdata/internal/datafiles" "github.com/larvit/fejkdata/internal/grammar" ) @@ -71,8 +71,8 @@ func compileInput(input string) (node, error) { // inputValue reads an inline template as the value compile takes: the JSON value it holds, the // whitespace around it dropped, or the input itself as a format string when it is not JSON. func inputValue(input string) any { - var raw any - if err := json.Unmarshal([]byte(input), &raw); err != nil { + raw, err := datafiles.DecodeJSON([]byte(input)) + if err != nil { return input } return raw diff --git a/internal/builtinfunc/builtins.go b/internal/builtinfunc/builtins.go index aac40ed..d2f53ac 100644 --- a/internal/builtinfunc/builtins.go +++ b/internal/builtinfunc/builtins.go @@ -186,9 +186,6 @@ func intRangeArgs(a []string) error { if lo > hi { return fmt.Errorf("int(min,max): min %d > max %d", lo, hi) } - if lo == hi { - return fmt.Errorf("int(%d,%d) is the constant %d; write it as text", lo, hi, lo) - } if uint64(hi)-uint64(lo) >= uint64(math.MaxInt64) { // span hi-lo+1 would overflow int -> IntN panic return fmt.Errorf("int(min,max): range %d..%d is too wide", lo, hi) } @@ -213,9 +210,6 @@ func floatArgs(a []string) error { if dp < 0 || dp > maxDecimals { return fmt.Errorf("float(min,max,dp): decimals %d out of range 0..%d", dp, maxDecimals) } - if lo == hi { - return fmt.Errorf("float(%s,%s,%d) is the constant %q; write it as text", a[0], a[1], dp, strconv.FormatFloat(lo, 'f', dp, 64)) - } if math.IsInf(hi-lo, 0) { // an overflowing span would render as "+Inf" return fmt.Errorf("float(min,max,dp): range %v..%v is too wide", lo, hi) } diff --git a/internal/builtinfunc/layout.go b/internal/builtinfunc/layout.go index 317d939..2765354 100644 --- a/internal/builtinfunc/layout.go +++ b/internal/builtinfunc/layout.go @@ -11,32 +11,12 @@ import ( const dayLayout = "2006-01-02" -// The instants a layout is proved against: layoutProbe2 is alike in no field, while -// layoutDay differs from layoutProbe in its date fields alone and layoutClock in its -// clock fields alone, so formatting two of them tells which kind a layout names. -var ( - layoutProbe = time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC) - layoutProbe2 = time.Date(2010, 11, 12, 13, 14, 15, 0, time.UTC) - layoutDay = time.Date(2010, 11, 12, 4, 5, 6, 0, time.UTC) - layoutClock = time.Date(2001, 2, 3, 13, 14, 15, 0, time.UTC) -) - -func namesAField(layout string) bool { - return layoutProbe.Format(layout) != layoutProbe2.Format(layout) -} -func namesADateField(layout string) bool { - return layoutProbe.Format(layout) != layoutDay.Format(layout) -} -func namesAClockField(layout string) bool { - return layoutProbe.Format(layout) != layoutClock.Format(layout) -} - // quotedLayout reports whether an arg carries the single quotes a layout is written in. func quotedLayout(a string) bool { return len(a) >= 2 && a[0] == '\'' && a[len(a)-1] == '\'' } -// layoutArg is the Go layout a quoted arg holds, refused when unquoted or constant. +// layoutArg is the Go layout a quoted arg holds, refused when unquoted. // docs/decisions.md#a-layout-is-always-quoted func layoutArg(a string) (string, error) { if !quotedLayout(a) { @@ -46,11 +26,7 @@ func layoutArg(a string) (string, error) { } return "", fmt.Errorf("layout %s is not quoted; write '%s'", a, bare) } - layout := a[1 : len(a)-1] - if !namesAField(layout) { - return "", fmt.Errorf("layout '%s' names no field, so it is the constant %q; write it as text", layout, layout) - } - return layout, nil + return a[1 : len(a)-1], nil } // layoutOf is layoutArg for an arg a check already validated. @@ -100,30 +76,15 @@ func dateArgs(a []string) error { if to.Before(from) { return fmt.Errorf("date(from,to,layout): from %s is after to %s", a[0], a[1]) } - layout, err := layoutArg(a[2]) - if err != nil { - return err - } - if !namesADateField(layout) { - return fmt.Errorf("date(from,to,layout): '%s' names no date field; write time('%s')", layout, layout) - } - if from.Equal(to) && !namesAClockField(layout) { - return fmt.Errorf("date(%s,%s,'%s') is the constant %q; write it as text", a[0], a[1], layout, from.Format(layout)) - } - return nil + _, err = layoutArg(a[2]) + return err } func timeArg(a []string) error { if err := layoutArity("time", 1, a); err != nil { return err } - layout, err := layoutArg(a[0]) - if err != nil { - return err - } - if namesADateField(layout) { - return fmt.Errorf("time(layout): '%s' names a date field; write date(from,to,layout)", layout) - } - return nil + _, err := layoutArg(a[0]) + return err } // datePrep draws a second in [from 00:00:00, to 23:59:59] UTC; the span is counted diff --git a/internal/datafiles/datafiles.go b/internal/datafiles/datafiles.go index b260fa6..6ae5b73 100644 --- a/internal/datafiles/datafiles.go +++ b/internal/datafiles/datafiles.go @@ -2,6 +2,7 @@ package datafiles import ( + "bytes" "encoding/json" "fmt" "io/fs" @@ -38,8 +39,7 @@ type Category struct { rows *rowsFiles } -// ReadRows reads a rows file beside the category and marks it as named; Walk refuses a rows -// file no category read. +// ReadRows reads a rows file beside the category. func (c Category) ReadRows(name string) (string, error) { return c.rows.read(name) } func (s Source) labelled(p string) string { @@ -101,11 +101,6 @@ func (s Source) walkDir(dir []string, compile func(Category) error) (bool, error } handed = handed || holds } - for name, named := range rows.tsv { - if !named && !strings.HasPrefix(name, ".") { - return false, fmt.Errorf("%s: no category names it in its rows; a table's rows file sits beside a category file naming it", s.labelled(path.Join(full, name))) - } - } return handed, nil } @@ -147,8 +142,8 @@ func (s Source) compileFile(dir []string, file string, rows *rowsFiles, compile if err != nil { return false, fmt.Errorf("%s: %w", s.labelled(full), err) } - var raw any - if err := json.Unmarshal(b, &raw); err != nil { + raw, err := DecodeJSON(b) + if err != nil { return false, fmt.Errorf("%s: %w", s.labelled(full), err) } if err := compile(Category{Folders: dir, Name: name, JSON: raw, rows: rows}); err != nil { @@ -157,8 +152,7 @@ func (s Source) compileFile(dir []string, file string, rows *rowsFiles, compile return true, nil } -// rowsFiles is what a category may name beside itself: the rows files of its folder, -// each marked once a category names it. +// rowsFiles is what a category may name beside itself: the rows files of its folder. type rowsFiles struct { src Source dir string @@ -169,20 +163,30 @@ func newRowsFiles(src Source, dir string, entries []fs.DirEntry) *rowsFiles { files := &rowsFiles{src: src, dir: dir, tsv: map[string]bool{}} for _, e := range entries { if strings.HasSuffix(e.Name(), ".tsv") && !e.IsDir() { - files.tsv[e.Name()] = false + files.tsv[e.Name()] = true } } return files } func (r *rowsFiles) read(name string) (string, error) { - if _, present := r.tsv[name]; !present { + if !r.tsv[name] { return "", fmt.Errorf("rows names %s, which is not beside it in %s", name, r.src.labelled(r.dir)) } - r.tsv[name] = true b, err := fs.ReadFile(r.src.fsys, path.Join(r.dir, name)) if err != nil { return "", fmt.Errorf("%s: %w", r.src.labelled(path.Join(r.dir, name)), err) } return string(b), nil } + +// DecodeJSON decodes one JSON value, keeping each number as the json.Number it is written as. +func DecodeJSON(b []byte) (any, error) { + var v any + if !json.Valid(b) { + return nil, json.Unmarshal(b, &v) + } + dec := json.NewDecoder(bytes.NewReader(b)) + dec.UseNumber() + return v, dec.Decode(&v) +} diff --git a/internal/drawstate/drawstate.go b/internal/drawstate/drawstate.go index a4550a7..ce6cc1c 100644 --- a/internal/drawstate/drawstate.go +++ b/internal/drawstate/drawstate.go @@ -21,10 +21,15 @@ func (s *State) IntN(n int) int { return s.rand.IntN(n) } func (s *State) Float64() float64 { return s.rand.Float64() } -// Weighted draws an index into cum, a running sum of positive weights, with odds by weight. +// Weighted draws an index into cum, a running sum of weights of 0 or more with a positive total, +// with odds by weight, so an index of weight 0 is never drawn. func (s *State) Weighted(cum []float64) int { - x := s.Float64() * cum[len(cum)-1] - return min(sort.Search(len(cum), func(i int) bool { return cum[i] > x }), len(cum)-1) // x can round up to the total + total := cum[len(cum)-1] + x := s.Float64() * total + if i := sort.Search(len(cum), func(i int) bool { return cum[i] > x }); i < len(cum) { + return i + } + return sort.Search(len(cum), func(i int) bool { return cum[i] >= total }) // x rounded up to the total } // Seq advances the counter {seq(key)} reads, or {seq()} for an empty key, and returns it. diff --git a/internal/rows/rows.go b/internal/rows/rows.go index a3a08b2..23a7bdf 100644 --- a/internal/rows/rows.go +++ b/internal/rows/rows.go @@ -209,8 +209,8 @@ func (t *Table[O]) proveKeys() error { return nil } -// sumWeights proves every weight is a positive number, and keeps the weights and their -// running sum. +// sumWeights proves every weight is a number of 0 or more, some above 0, and keeps the weights +// and their running sum. func (t *Table[O]) sumWeights() error { if t.weightIndex < 0 { return nil @@ -219,14 +219,39 @@ func (t *Table[O]) sumWeights() error { total := 0.0 for r := range t.cum { w, err := strconv.ParseFloat(t.Cell(r, t.weightIndex), 64) - if err != nil || math.IsInf(w, 0) || math.IsNaN(w) || w <= 0 { - return fmt.Errorf("%s line %d: weight %q is not a positive number", t.file, r+2, t.Cell(r, t.weightIndex)) + if err != nil || math.IsInf(w, 0) || math.IsNaN(w) || w < 0 { + return fmt.Errorf("%s line %d: weight %q is not a number of 0 or more", t.file, r+2, t.Cell(r, t.weightIndex)) } total += w t.weights[r], t.cum[r] = w, total } - if math.IsInf(total, 0) { + switch { + case math.IsInf(total, 0): return fmt.Errorf("%s: the weights sum past the largest number", t.file) + case total == 0: + return fmt.Errorf("%s: every weight is 0, so no row is drawn", t.file) + } + return t.proveWeightUnderEachParent() +} + +// proveWeightUnderEachParent proves each parent key's rows weigh more than 0 together, so a draw +// inside that parent's row has a row to draw. +func (t *Table[O]) proveWeightUnderEachParent() error { + if t.parentIndex < 0 { + return nil + } + sums, keys := map[string]float64{}, []string{} + for r := range t.weights { + k := t.Cell(r, t.parentIndex) + if _, seen := sums[k]; !seen { + keys = append(keys, k) + } + sums[k] += t.weights[r] + } + for _, k := range keys { + if sums[k] == 0 { + return fmt.Errorf("%s: every row under %s %q weighs 0, so none is drawn there", t.file, t.header[t.parentIndex], k) + } } return nil } diff --git a/name.go b/name.go index 8c53d06..918aa28 100644 --- a/name.go +++ b/name.go @@ -57,8 +57,7 @@ func (sc *nameScope) lookup(name string) *nameBinding { // bindNames gives every template of a compiled category or inline template the scope its // names live in, and refuses a name bound twice along one chain of scopes, a binding in a -// choice's item, a name a field spells too, a read no field or name answers, and a binding -// read by nothing. +// choice's item, a name a field spells too, and a read no field or name answers. func bindNames(root node) error { var scopes []*nameScope var gather func(n node, scope *nameScope, inChoice bool, where string) error @@ -90,22 +89,7 @@ func bindNames(root node) error { } } } - read, err := answerReads(root, scopes) - if err != nil { - return err - } - return checkEveryNameRead(scopes, read) -} - -func checkEveryNameRead(scopes []*nameScope, read map[*nameBinding]bool) error { - for _, sc := range scopes { - for _, b := range sc.order { - if !read[b] { - return fmt.Errorf("%stoken {%s}: nothing reads name %q; drop the token", b.where, b.body, b.name) - } - } - } - return nil + return answerReads(root, scopes) } func (sc *nameScope) bindAll(t *template, inChoice bool, where string) error { @@ -161,12 +145,10 @@ func eachContained(n node, where string, fn func(c node, where string) error) er return nil } -// answerReads answers each unbound read under root with a name its template sees, and returns -// the set of bindings some read answers to. It answers a template's fields before its format, the +// answerReads proves each unbound read under root is answered by a name its template sees. It answers a template's fields before its format, the // order compile reports in, and refuses a field spelling a name. A refusal searches scopes, every // scope of the category, for a name bound where the read cannot see it. -func answerReads(root node, scopes []*nameScope) (map[*nameBinding]bool, error) { - read := map[*nameBinding]bool{} +func answerReads(root node, scopes []*nameScope) error { var answer func(n node, where string) error answer = func(n node, where string) error { t, isTemplate := n.(*template) @@ -186,15 +168,13 @@ func answerReads(root node, scopes []*nameScope) (map[*nameBinding]bool, error) } } for _, u := range t.unbound { - b := t.nameScope.lookup(u.head) - if b == nil { + if t.nameScope.lookup(u.head) == nil { return unanswered(where, u, t.nameScope, scopes) } - read[b] = true } return nil } - return read, answer(root, "") + return answer(root, "") } // unanswered is the refusal of u, a read no field or name answers: naming the repeat binding the diff --git a/node.go b/node.go index 23f84e2..553420d 100644 --- a/node.go +++ b/node.go @@ -1,11 +1,14 @@ package fejkdata import ( + "encoding/json" "fmt" "math" + "strconv" "github.com/larvit/fejkdata/internal/datafiles" "github.com/larvit/fejkdata/internal/grammar" + "github.com/larvit/fejkdata/internal/invariant" ) // node is a compiled element of the namespace tree: a folder, choice, null, @@ -87,7 +90,7 @@ func (t *template) startOf(name string) node { // compile converts parsed JSON — a category or an inline template — into a node tree, // validating structure up front. func compile(v any) (node, error) { - n, err := compileAt(v, atTop) + n, err := compileItem(v, atTop) if err != nil { return nil, err } @@ -115,22 +118,11 @@ const ( inColumn // a column, or a choice item standing in for one ) -// compileAt compiles a node drawn against no other: no choice's item, or a one-item choice's. -// A weight there does nothing, bar the default 1 written out. -func compileAt(v any, pos position) (node, error) { - if o, isObject := v.(map[string]any); isObject { - if w, isNumber := o["weight"].(float64); isNumber && w != 1 { - return nil, fmt.Errorf(`weight %v skews a draw between items, and nothing here is drawn against this one; drop "weight"`, w) - } - } +// compileItem compiles one node; a weight on it skews only a choice drawing it. +func compileItem(v any, pos position) (node, error) { if _, err := weightOf(v); err != nil { return nil, err } - return compileItem(v, pos) -} - -// compileItem compiles one node, allowing the weight a choice item may carry. -func compileItem(v any, pos position) (node, error) { switch v := v.(type) { case string: t, err := compileString(v) @@ -144,19 +136,22 @@ func compileItem(v any, pos position) (node, error) { return compileTemplate(v, pos) case nil: if pos != inColumn { - return nil, fmt.Errorf(`null is a record column's value, and here it only renders "", so write ""`) + return compileString("") } return &nullItem{}, nil - default: - return nil, fmt.Errorf("a template value must be a string, a list or an object, not %s", jsonKind(v)) + case json.Number: + return compileString(string(v)) + case bool: + return compileString(strconv.FormatBool(v)) } + panic(invariant.Broken("compileItem has no case for the JSON value %T", v)) } -// jsonKind names a JSON value a template cannot hold, in the data format's own +// jsonKind names the kind of a JSON value an option cannot take, in the data format's own // terms rather than the decoding library's. func jsonKind(v any) string { switch v.(type) { - case float64: + case json.Number: return "a number" case bool: return "a boolean" @@ -198,7 +193,7 @@ func compileChoice(items []any, pos position) (node, error) { case 0: return nil, fmt.Errorf("empty choice") case 1: - return compileAt(items[0], pos) + return compileItem(items[0], pos) } c := &choice{items: make([]node, len(items))} cum := make([]float64, len(items)) @@ -224,6 +219,9 @@ func compileChoice(items []any, pos position) (node, error) { if math.IsInf(total, 1) { return nil, fmt.Errorf("choice weights must sum to a finite number, got %v", total) } + if total == 0 { + return nil, fmt.Errorf("every weight is 0, so the choice has nothing to draw") + } c.cum = cum } c.shared = sharedPaths(c.items, true) @@ -280,9 +278,6 @@ func readOptions(m map[string]any, pos position) (templateOptions, error) { if o.separator, ok = sv.(string); !ok { return o, fmt.Errorf("separator must be a string, got %T", sv) } - if repeat == 1 && o.separator != "" { - return o, fmt.Errorf(`separator %q joins repeated renders, and this template renders once; drop "separator", or add a repeat above 1`, o.separator) - } } return o, nil } @@ -296,7 +291,7 @@ func compileFields(m map[string]any, pos position) (map[string]node, error) { if err := grammar.CheckIdentifier(k); err != nil { return nil, fmt.Errorf("field %w", err) } - n, err := compileAt(m[k], pos) + n, err := compileItem(m[k], pos) if err != nil { return nil, fmt.Errorf("field %q: %w", k, err) } @@ -312,9 +307,9 @@ func repeatOf(m map[string]any) (int, error) { if !ok { return 1, nil } - r, ok := rv.(float64) + r, ok := numberOf(rv) if !ok { - return 0, fmt.Errorf("repeat must be a number, got %T", rv) + return 0, fmt.Errorf("repeat must be a number, not %s", jsonKind(rv)) } if math.IsNaN(r) || math.IsInf(r, 0) || r < 1 || r != math.Trunc(r) { return 0, fmt.Errorf("repeat must be a positive integer, got %v", rv) @@ -326,7 +321,7 @@ func repeatOf(m map[string]any) (int, error) { } // weightOf reads a node's "weight" (default 1) from its raw JSON form. Only -// template objects carry weight; a present one must be finite and positive. +// template objects carry weight; a present one must be finite and not negative. func weightOf(raw any) (float64, error) { m, ok := raw.(map[string]any) if !ok { @@ -336,19 +331,26 @@ func weightOf(raw any) (float64, error) { if !ok { return 1, nil } - w, ok := wv.(float64) + w, ok := numberOf(wv) if !ok { return 0, fmt.Errorf(`weight is an option and takes a number, not %s; a field cannot be named "weight"`, jsonKind(wv)) } - if w < 0 || math.IsNaN(w) || math.IsInf(w, 0) { - return 0, fmt.Errorf("weight must be finite and positive, got %v", w) - } - if w == 0 { - return 0, fmt.Errorf("weight 0 means the item is never drawn; remove the item instead") + if w < 0 || math.IsInf(w, 0) { + return 0, fmt.Errorf("weight must be a finite number of 0 or more, got %v", wv) } return w, nil } +// numberOf reads a JSON number, one too large for a float64 as an infinity. +func numberOf(v any) (float64, bool) { + n, ok := v.(json.Number) + if !ok { + return 0, false + } + f, _ := n.Float64() + return f, true +} + func isOption(name string) bool { switch name { case "datatype", "format", "repeat", "separator", "weight": diff --git a/struct.go b/struct.go index 1990f53..1826866 100644 --- a/struct.go +++ b/struct.go @@ -56,9 +56,6 @@ func (f *Generator) structShapeOf(t reflect.Type) (*structShape, error) { } sc := &structCompile{root: &f.root, visiting: map[reflect.Type]bool{}, structsLeft: maxStructs} shape, err := sc.compileShape(t, label) - if err == nil && shape.empty() { - err = fmt.Errorf("%s has no fake tags, so nothing to fill", t) - } if f.structs == nil { f.structs = map[reflect.Type]structResult{} }