Add record output to JSON, CSV and SQL #8
@@ -95,15 +95,26 @@ the whole. `--format json|csv|sql` streams a record per line; the library's
|
||||
```sh
|
||||
fejkdata --format json users # {"first":"Ada","last":"Lovelace"}
|
||||
fejkdata --format csv users # first,last → Ada,Lovelace
|
||||
fejkdata --format sql users # INSERT INTO users (first, last) VALUES ('Ada', 'Lovelace');
|
||||
fejkdata --format sql users # INSERT INTO "users" ("first", "last") VALUES ('Ada', 'Lovelace');
|
||||
fejkdata --format json --repeat 3 users # three objects, one per line
|
||||
fejkdata --format sql --table people users # INSERT into another table
|
||||
```
|
||||
|
||||
`--repeat` streams that many records — a JSON object per line, a CSV row per
|
||||
line after a header, an INSERT per line in SQL. Columns render independently, each
|
||||
a fresh draw, in name order. A column is a string, and each format quotes it as
|
||||
such; see [Decisions](#decisions) for the typed-scalar and struct-filling scope.
|
||||
line after a header, an INSERT per line in SQL. Only a category-level template is
|
||||
a record; a field, choice or folder errors. Column identifiers are double-quoted
|
||||
in SQL, so a hyphenated field like `postal-code` stays valid. A column is a
|
||||
string, and each format quotes it as such; see [Decisions](#decisions) for the
|
||||
typed-scalar, correlation and struct-filling scope.
|
||||
|
||||
A record written only to emit columns still needs a `format` — the grammar's one
|
||||
required key — so `"format": ""` carries the fields with an inert format: it
|
||||
renders nothing by `Fake`, and is compiled only so the tree's fences still run.
|
||||
The columns are the point, and their facts stay together: two columns that
|
||||
reference one category — `{/currency.code}` and `{/currency.symbol}` — share one
|
||||
draw of it, so the record is internally consistent. A field hold, transform or
|
||||
operand ties fields together within one column as always (see
|
||||
[Correlated fields](#correlated-fields) and [Decisions](#decisions)).
|
||||
|
||||
## Library
|
||||
|
||||
@@ -498,13 +509,21 @@ tokens add cost in proportion to the output.
|
||||
template's `format` composes its fields into one string; `Record` and
|
||||
`--format` project the same fields as columns. Two views of one dataset, so a
|
||||
record author writes the same JSON they already know, and a column is the same
|
||||
field `Fake` renders by dotted path.
|
||||
field `Fake` renders by dotted path. The `format` is inert to a record — a
|
||||
record-only template writes `"format": ""` — but it is compiled and fenced, so
|
||||
a template that loads renders as whichever shape is asked for.
|
||||
- **Records emit text; typed scalars are out.** Every value fejkdata yields is a
|
||||
string, so JSON, CSV and SQL each quote a column as text (`"42"`, `'42'`) rather
|
||||
than guess a number or a boolean. Emitting unquoted numbers or booleans would
|
||||
need a per-column `kind`, a parallel scalar system in a format whose promise is
|
||||
"text means what it says". A column's check digit, number or id is still
|
||||
valid-by-construction through a builtin; it is serialized as text.
|
||||
- **A record's columns share one reference draw.** Two columns that reference one
|
||||
category — `{/currency.code}` beside `{/currency.symbol}` — read one draw of it,
|
||||
so a record's facts agree the way a template's [correlated
|
||||
fields](#correlated-fields) do. Only references share across columns: a sibling
|
||||
field is local to its own column, so `first` does not silently bind to a
|
||||
`first` in the column next to it.
|
||||
- **Filling a Go struct is out of scope.** `Record.Fields()` returns the columns a
|
||||
caller maps onto a struct themselves. gofakeit's `fake:"{firstname}"` tags
|
||||
reflect over an arbitrary struct type and cast into its fields — a different
|
||||
|
||||
@@ -503,14 +503,14 @@ func TestRunRecordSQL(t *testing.T) {
|
||||
t.Fatalf("run = %d, stderr=%q", code, errb)
|
||||
}
|
||||
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
|
||||
if len(lines) != 2 || !strings.HasPrefix(lines[0], "INSERT INTO people (first, last) VALUES (") {
|
||||
if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) {
|
||||
t.Fatalf("sql output = %q, want two INSERT INTO people statements", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRecordDefaultTable(t *testing.T) {
|
||||
code, out, errb := runOut("--seed", "1", "--format", "sql", "--data-path", recordDir(t), "users")
|
||||
if code != 0 || !strings.HasPrefix(out, "INSERT INTO users (") {
|
||||
if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
|
||||
t.Fatalf("default table = %d, %q, want the path's last segment; stderr %q", code, out, errb)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -267,14 +267,20 @@ type draws struct {
|
||||
// gives one value, and a shown operand is the operand computed. Every other name is
|
||||
// drawn afresh, so {word} {word} still draws twice. checkTokens, checkPath and
|
||||
// linkRefs prove every step, so the walk cannot fail.
|
||||
func readField(s *session, t *template, held *draws, a arm) string {
|
||||
func readField(s *session, t *template, held, shared *draws, a arm) string {
|
||||
if !t.held[a.key] {
|
||||
if len(a.tail) > 0 {
|
||||
panic(fmt.Sprintf("fejkdata: %q reads a path into %q, which the expansion does not hold", a.name, a.key))
|
||||
}
|
||||
return render(s, t.fields[a.key])
|
||||
return renderShared(s, t.fields[a.key], shared)
|
||||
}
|
||||
if v, read := held.value[a.path]; read {
|
||||
// A reference names a shared source, so a record shares its draw across the
|
||||
// columns; a sibling field is drawn per expansion as always.
|
||||
d := held
|
||||
if isRef(a.key) && shared != nil {
|
||||
d = shared
|
||||
}
|
||||
if v, read := d.value[a.path]; read {
|
||||
return v
|
||||
}
|
||||
var v string
|
||||
@@ -286,16 +292,16 @@ func readField(s *session, t *template, held *draws, a arm) string {
|
||||
if consumed := len(a.tail) - len(rest); consumed > 0 {
|
||||
key = a.steps[consumed-1]
|
||||
}
|
||||
n, drew := held.variant[key]
|
||||
n, drew := d.variant[key]
|
||||
if !drew {
|
||||
n = drawn(s, c)
|
||||
held.variant[key] = n
|
||||
d.variant[key] = n
|
||||
}
|
||||
return []node{n}, nil
|
||||
},
|
||||
leaf: func(n node) error { v = render(s, n); return nil },
|
||||
leaf: func(n node) error { v = renderShared(s, n, shared); return nil },
|
||||
})
|
||||
held.value[a.path] = v
|
||||
d.value[a.path] = v
|
||||
return v
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -718,6 +718,6 @@ func TestReadFieldPanicsOnAnUnheldPath(t *testing.T) {
|
||||
t.Fatal("not a template")
|
||||
}
|
||||
mustPanic(t, "unheld arm with a path", func() {
|
||||
readField(engine(1).rand, tm, nil, arm{name: "w.x", key: "w", tail: []string{"x"}, path: "w.x"})
|
||||
readField(engine(1).rand, tm, nil, nil, arm{name: "w.x", key: "w", tail: []string{"x"}, path: "w.x"})
|
||||
})
|
||||
}
|
||||
|
||||
+1
-1
@@ -63,7 +63,7 @@ func TestReadmeRecordExample(t *testing.T) {
|
||||
if len(got) != 2 || got[0].Name != "first" || got[1].Name != "last" {
|
||||
t.Fatalf("record columns = %v, want first, last", got)
|
||||
}
|
||||
if r.CSVHeader() != "first,last" || !strings.HasPrefix(r.SQLInsert("users"), "INSERT INTO users (first, last) VALUES (") {
|
||||
if r.CSVHeader() != "first,last" || !strings.HasPrefix(r.SQLInsert("users"), `INSERT INTO "users" ("first", "last") VALUES (`) {
|
||||
t.Fatalf("serializers = %q, %q, want first,last and an INSERT", r.CSVHeader(), r.SQLInsert("users"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,16 +71,20 @@ func csvLine(cols []string) string {
|
||||
return strings.TrimSuffix(b.String(), "\n")
|
||||
}
|
||||
|
||||
// SQLInsert renders the record as one INSERT statement into table, every column a
|
||||
// single-quoted string literal.
|
||||
// SQLInsert renders the record as one INSERT statement into table: identifiers in
|
||||
// ANSI double quotes, every value a single-quoted string literal.
|
||||
func (r *Record) SQLInsert(table string) string {
|
||||
cols := make([]string, len(r.fields))
|
||||
vals := make([]string, len(r.fields))
|
||||
for i, f := range r.fields {
|
||||
cols[i] = f.Name
|
||||
cols[i] = quoteIdent(f.Name)
|
||||
vals[i] = "'" + strings.ReplaceAll(f.Value, "'", "''") + "'"
|
||||
}
|
||||
return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);", table, strings.Join(cols, ", "), strings.Join(vals, ", "))
|
||||
return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);", quoteIdent(table), strings.Join(cols, ", "), strings.Join(vals, ", "))
|
||||
}
|
||||
|
||||
func quoteIdent(s string) string {
|
||||
return `"` + strings.ReplaceAll(s, `"`, `""`) + `"`
|
||||
}
|
||||
|
||||
// Record renders a path as one record: the template it names, with each direct
|
||||
@@ -98,7 +102,7 @@ func (f *Generator) Record(path string) (*Record, error) {
|
||||
}
|
||||
t, ok := n.(*template)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("fejkdata: %s does not name a record; it has no fields to project", path)
|
||||
return nil, fmt.Errorf("fejkdata: %s names a choice, not a template; only a category-level template is a record", path)
|
||||
}
|
||||
r := renderRecord(f.rand, t)
|
||||
if len(r.fields) == 0 {
|
||||
@@ -156,11 +160,13 @@ func (f *Generator) FakeRecord(input string) (*Record, error) {
|
||||
|
||||
// renderRecord projects a template's direct fields as columns, drawn once each,
|
||||
// in name order. A {/path} binding is a render edge, not a column, so it is
|
||||
// skipped the same way List and the graph do.
|
||||
// skipped the same way List and the graph do. The columns share one draw context,
|
||||
// so two columns that reference one category read one draw of it.
|
||||
func renderRecord(s *session, t *template) *Record {
|
||||
shared := &draws{variant: map[string]node{}, value: map[string]string{}}
|
||||
r := &Record{}
|
||||
for _, name := range recordColumns(t) {
|
||||
r.fields = append(r.fields, Field{Name: name, Value: render(s, t.fields[name])})
|
||||
r.fields = append(r.fields, Field{Name: name, Value: renderShared(s, t.fields[name], shared)})
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
+48
-3
@@ -73,7 +73,7 @@ func TestRecordErrors(t *testing.T) {
|
||||
want string
|
||||
}{
|
||||
{"bare", "has no fields, so no columns"},
|
||||
{"pick", "does not name a record"},
|
||||
{"pick", "names a choice"},
|
||||
{"group", "names a folder"},
|
||||
{"nope", "no entry"},
|
||||
} {
|
||||
@@ -134,8 +134,53 @@ func TestRecordSQLInsert(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := r.SQLInsert("people")
|
||||
if got != "INSERT INTO people (last) VALUES ('O''Brien');" {
|
||||
t.Fatalf("SQLInsert() = %q, want the single quote doubled", got)
|
||||
if got != `INSERT INTO "people" ("last") VALUES ('O''Brien');` {
|
||||
t.Fatalf("SQLInsert() = %q, want quoted identifiers and the single quote doubled", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordSharesAReferenceAcrossColumns(t *testing.T) {
|
||||
dir := writeData(t, map[string]string{
|
||||
"currency": `[{"format":"{code}","code":"AUD","symbol":"$"},{"format":"{code}","code":"EUR","symbol":"€"}]`,
|
||||
"price": `{"format":"","code":"{/currency.code}","symbol":"{/currency.symbol}"}`,
|
||||
})
|
||||
f := newGenerator(t, dir, WithSeed(1))
|
||||
for i := 0; i < 100; i++ {
|
||||
r, err := f.Record("price")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := map[string]string{}
|
||||
for _, c := range r.Fields() {
|
||||
m[c.Name] = c.Value
|
||||
}
|
||||
switch m["code"] {
|
||||
case "AUD":
|
||||
if m["symbol"] != "$" {
|
||||
t.Fatalf("record %q: code AUD but symbol %q, want one currency draw across columns", r.JSON(), m["symbol"])
|
||||
}
|
||||
case "EUR":
|
||||
if m["symbol"] != "€" {
|
||||
t.Fatalf("record %q: code EUR but symbol %q, want one currency draw across columns", r.JSON(), m["symbol"])
|
||||
}
|
||||
default:
|
||||
t.Fatalf("record %q has unexpected code %q", r.JSON(), m["code"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordSQLQuotesIdentifiers(t *testing.T) {
|
||||
dir := writeData(t, map[string]string{
|
||||
"row": `{"format": "", "postal-code": "1", "street-number": "2"}`,
|
||||
})
|
||||
f := newGenerator(t, dir, WithSeed(1))
|
||||
r, err := f.Record("row")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := r.SQLInsert("my-table")
|
||||
if got != `INSERT INTO "my-table" ("postal-code", "street-number") VALUES ('1', '2');` {
|
||||
t.Fatalf("SQLInsert() = %q, want hyphenated identifiers and table quoted", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -52,15 +52,22 @@ func descend(s *session, root node, segments []string) (node, error) {
|
||||
// render evaluates a compiled node to a string. compile validates every node up
|
||||
// front, so rendering a compiled tree cannot fail.
|
||||
func render(s *session, n node) string {
|
||||
return renderShared(s, n, nil)
|
||||
}
|
||||
|
||||
// renderShared is render with a shared draw context: the draws a record shares
|
||||
// across its columns. A nil shared means a standalone render, where a reference
|
||||
// is drawn per expansion as it always has been.
|
||||
func renderShared(s *session, n node, shared *draws) string {
|
||||
switch n := n.(type) {
|
||||
case *choice:
|
||||
return render(s, pick(s, n))
|
||||
return renderShared(s, pick(s, n), shared)
|
||||
case *template:
|
||||
if n.repeat == 1 {
|
||||
if n.fixed {
|
||||
return n.lit
|
||||
}
|
||||
return expand(s, n)
|
||||
return expand(s, n, shared)
|
||||
}
|
||||
var b strings.Builder
|
||||
b.Grow(n.repeat * (n.grow + len(n.separator)))
|
||||
@@ -68,7 +75,7 @@ func render(s *session, n node) string {
|
||||
if i > 0 {
|
||||
b.WriteString(n.separator)
|
||||
}
|
||||
b.WriteString(expand(s, n))
|
||||
b.WriteString(expand(s, n, shared))
|
||||
}
|
||||
return b.String()
|
||||
default:
|
||||
@@ -90,11 +97,12 @@ func pick(r rng, c *choice) node {
|
||||
|
||||
// expand renders a template's compiled ops. compile validated every token, so this
|
||||
// cannot fail.
|
||||
func expand(s *session, t *template) string {
|
||||
func expand(s *session, t *template, shared *draws) string {
|
||||
var b strings.Builder
|
||||
b.Grow(t.grow)
|
||||
// One draw per held name, for this expansion only: a nested template and each
|
||||
// repeat iteration get their own, since each is its own expansion.
|
||||
// repeat iteration get their own, since each is its own expansion. A shared
|
||||
// draw context, when a record supplies one, overrides that for references.
|
||||
var held *draws
|
||||
if len(t.held) > 0 {
|
||||
held = &draws{
|
||||
@@ -108,7 +116,7 @@ func expand(s *session, t *template) string {
|
||||
case 'l':
|
||||
b.WriteString(o.lit)
|
||||
case 'f':
|
||||
b.WriteString(readField(s, t, held, o.arms[s.IntN(len(o.arms))]))
|
||||
b.WriteString(readField(s, t, held, shared, o.arms[s.IntN(len(o.arms))]))
|
||||
case 'b':
|
||||
// Read before the call, so the value a calc computes is the value the
|
||||
// format showed. calcVars fixed the order op.operands holds.
|
||||
@@ -116,7 +124,7 @@ func expand(s *session, t *template) string {
|
||||
if len(o.operands) > 0 {
|
||||
operands = make([]string, len(o.operands))
|
||||
for j, a := range o.operands {
|
||||
operands[j] = readField(s, t, held, a)
|
||||
operands[j] = readField(s, t, held, shared, a)
|
||||
}
|
||||
}
|
||||
b.WriteString(o.call(s, b.String(), operands)) // b.String() is the output so far
|
||||
|
||||
+1
-1
@@ -315,7 +315,7 @@ func TestGrowIsALowerBound(t *testing.T) {
|
||||
t.Fatalf("format %q did not compile to a template", format)
|
||||
}
|
||||
for i := 0; i < 50; i++ {
|
||||
if got := len(expand(f.rand, tmpl)); got < tmpl.grow {
|
||||
if got := len(expand(f.rand, tmpl, nil)); got < tmpl.grow {
|
||||
t.Errorf("format %q: expand emitted %d bytes, below grow %d", format, got, tmpl.grow)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user