Add record output to JSON, CSV and SQL #8

Open
lilleman wants to merge 21 commits from record-output into main
9 changed files with 121 additions and 37 deletions
Showing only changes of commit 6a8aa96b20 - Show all commits
+27 -8
View File
@@ -93,17 +93,28 @@ the whole. `--format json|csv|sql` streams a record per line; the library's
```
```sh
fejkdata --format json users # {"first":"Ada","last":"Lovelace"}
fejkdata --format csv users # first,last → Ada,Lovelace
fejkdata --format sql users # INSERT INTO users (first, last) VALUES ('Ada', 'Lovelace');
fejkdata --format json --repeat 3 users # three objects, one per line
fejkdata --format json users # {"first":"Ada","last":"Lovelace"}
fejkdata --format csv users # first,last → Ada,Lovelace
fejkdata --format sql users # INSERT INTO "users" ("first", "last") VALUES ('Ada', 'Lovelace');
fejkdata --format json --repeat 3 users # three objects, one per line
fejkdata --format sql --table people users # INSERT into another table
```
`--repeat` streams that many records — a JSON object per line, a CSV row per
line after a header, an INSERT per line in SQL. Columns render independently, each
a fresh draw, in name order. A column is a string, and each format quotes it as
such; see [Decisions](#decisions) for the typed-scalar and struct-filling scope.
line after a header, an INSERT per line in SQL. Only a category-level template is
a record; a field, choice or folder errors. Column identifiers are double-quoted
in SQL, so a hyphenated field like `postal-code` stays valid. A column is a
string, and each format quotes it as such; see [Decisions](#decisions) for the
typed-scalar, correlation and struct-filling scope.
A record written only to emit columns still needs a `format` — the grammar's one
required key — so `"format": ""` carries the fields with an inert format: it
renders nothing by `Fake`, and is compiled only so the tree's fences still run.
The columns are the point, and their facts stay together: two columns that
reference one category — `{/currency.code}` and `{/currency.symbol}` — share one
draw of it, so the record is internally consistent. A field hold, transform or
operand ties fields together within one column as always (see
[Correlated fields](#correlated-fields) and [Decisions](#decisions)).
## Library
@@ -498,13 +509,21 @@ tokens add cost in proportion to the output.
template's `format` composes its fields into one string; `Record` and
`--format` project the same fields as columns. Two views of one dataset, so a
record author writes the same JSON they already know, and a column is the same
field `Fake` renders by dotted path.
field `Fake` renders by dotted path. The `format` is inert to a record — a
record-only template writes `"format": ""` — but it is compiled and fenced, so
a template that loads renders as whichever shape is asked for.
- **Records emit text; typed scalars are out.** Every value fejkdata yields is a
string, so JSON, CSV and SQL each quote a column as text (`"42"`, `'42'`) rather
than guess a number or a boolean. Emitting unquoted numbers or booleans would
need a per-column `kind`, a parallel scalar system in a format whose promise is
"text means what it says". A column's check digit, number or id is still
valid-by-construction through a builtin; it is serialized as text.
- **A record's columns share one reference draw.** Two columns that reference one
category — `{/currency.code}` beside `{/currency.symbol}` — read one draw of it,
so a record's facts agree the way a template's [correlated
fields](#correlated-fields) do. Only references share across columns: a sibling
field is local to its own column, so `first` does not silently bind to a
`first` in the column next to it.
- **Filling a Go struct is out of scope.** `Record.Fields()` returns the columns a
caller maps onto a struct themselves. gofakeit's `fake:"{firstname}"` tags
reflect over an arbitrary struct type and cast into its fields — a different
+2 -2
View File
@@ -503,14 +503,14 @@ func TestRunRecordSQL(t *testing.T) {
t.Fatalf("run = %d, stderr=%q", code, errb)
}
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
if len(lines) != 2 || !strings.HasPrefix(lines[0], "INSERT INTO people (first, last) VALUES (") {
if len(lines) != 2 || !strings.HasPrefix(lines[0], `INSERT INTO "people" ("first", "last") VALUES (`) {
t.Fatalf("sql output = %q, want two INSERT INTO people statements", out)
}
}
func TestRunRecordDefaultTable(t *testing.T) {
code, out, errb := runOut("--seed", "1", "--format", "sql", "--data-path", recordDir(t), "users")
if code != 0 || !strings.HasPrefix(out, "INSERT INTO users (") {
if code != 0 || !strings.HasPrefix(out, `INSERT INTO "users" (`) {
t.Fatalf("default table = %d, %q, want the path's last segment; stderr %q", code, out, errb)
}
}
+13 -7
View File
@@ -267,14 +267,20 @@ type draws struct {
// gives one value, and a shown operand is the operand computed. Every other name is
// drawn afresh, so {word} {word} still draws twice. checkTokens, checkPath and
// linkRefs prove every step, so the walk cannot fail.
func readField(s *session, t *template, held *draws, a arm) string {
func readField(s *session, t *template, held, shared *draws, a arm) string {
if !t.held[a.key] {
if len(a.tail) > 0 {
panic(fmt.Sprintf("fejkdata: %q reads a path into %q, which the expansion does not hold", a.name, a.key))
}
return render(s, t.fields[a.key])
return renderShared(s, t.fields[a.key], shared)
}
if v, read := held.value[a.path]; read {
// A reference names a shared source, so a record shares its draw across the
// columns; a sibling field is drawn per expansion as always.
d := held
if isRef(a.key) && shared != nil {
d = shared
}
if v, read := d.value[a.path]; read {
return v
}
var v string
@@ -286,16 +292,16 @@ func readField(s *session, t *template, held *draws, a arm) string {
if consumed := len(a.tail) - len(rest); consumed > 0 {
key = a.steps[consumed-1]
}
n, drew := held.variant[key]
n, drew := d.variant[key]
if !drew {
n = drawn(s, c)
held.variant[key] = n
d.variant[key] = n
}
return []node{n}, nil
},
leaf: func(n node) error { v = render(s, n); return nil },
leaf: func(n node) error { v = renderShared(s, n, shared); return nil },
})
held.value[a.path] = v
d.value[a.path] = v
return v
}
+1 -1
View File
@@ -718,6 +718,6 @@ func TestReadFieldPanicsOnAnUnheldPath(t *testing.T) {
t.Fatal("not a template")
}
mustPanic(t, "unheld arm with a path", func() {
readField(engine(1).rand, tm, nil, arm{name: "w.x", key: "w", tail: []string{"x"}, path: "w.x"})
readField(engine(1).rand, tm, nil, nil, arm{name: "w.x", key: "w", tail: []string{"x"}, path: "w.x"})
})
}
+1 -1
View File
@@ -63,7 +63,7 @@ func TestReadmeRecordExample(t *testing.T) {
if len(got) != 2 || got[0].Name != "first" || got[1].Name != "last" {
t.Fatalf("record columns = %v, want first, last", got)
}
if r.CSVHeader() != "first,last" || !strings.HasPrefix(r.SQLInsert("users"), "INSERT INTO users (first, last) VALUES (") {
if r.CSVHeader() != "first,last" || !strings.HasPrefix(r.SQLInsert("users"), `INSERT INTO "users" ("first", "last") VALUES (`) {
t.Fatalf("serializers = %q, %q, want first,last and an INSERT", r.CSVHeader(), r.SQLInsert("users"))
}
}
+13 -7
View File
@@ -71,16 +71,20 @@ func csvLine(cols []string) string {
return strings.TrimSuffix(b.String(), "\n")
}
// SQLInsert renders the record as one INSERT statement into table, every column a
// single-quoted string literal.
// SQLInsert renders the record as one INSERT statement into table: identifiers in
// ANSI double quotes, every value a single-quoted string literal.
func (r *Record) SQLInsert(table string) string {
cols := make([]string, len(r.fields))
vals := make([]string, len(r.fields))
for i, f := range r.fields {
cols[i] = f.Name
cols[i] = quoteIdent(f.Name)
vals[i] = "'" + strings.ReplaceAll(f.Value, "'", "''") + "'"
}
return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);", table, strings.Join(cols, ", "), strings.Join(vals, ", "))
return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);", quoteIdent(table), strings.Join(cols, ", "), strings.Join(vals, ", "))
}
func quoteIdent(s string) string {
return `"` + strings.ReplaceAll(s, `"`, `""`) + `"`
}
// Record renders a path as one record: the template it names, with each direct
@@ -98,7 +102,7 @@ func (f *Generator) Record(path string) (*Record, error) {
}
t, ok := n.(*template)
if !ok {
return nil, fmt.Errorf("fejkdata: %s does not name a record; it has no fields to project", path)
return nil, fmt.Errorf("fejkdata: %s names a choice, not a template; only a category-level template is a record", path)
}
r := renderRecord(f.rand, t)
if len(r.fields) == 0 {
@@ -156,11 +160,13 @@ func (f *Generator) FakeRecord(input string) (*Record, error) {
// renderRecord projects a template's direct fields as columns, drawn once each,
// in name order. A {/path} binding is a render edge, not a column, so it is
// skipped the same way List and the graph do.
// skipped the same way List and the graph do. The columns share one draw context,
// so two columns that reference one category read one draw of it.
func renderRecord(s *session, t *template) *Record {
shared := &draws{variant: map[string]node{}, value: map[string]string{}}
r := &Record{}
for _, name := range recordColumns(t) {
r.fields = append(r.fields, Field{Name: name, Value: render(s, t.fields[name])})
r.fields = append(r.fields, Field{Name: name, Value: renderShared(s, t.fields[name], shared)})
}
return r
}
+48 -3
View File
@@ -73,7 +73,7 @@ func TestRecordErrors(t *testing.T) {
want string
}{
{"bare", "has no fields, so no columns"},
{"pick", "does not name a record"},
{"pick", "names a choice"},
{"group", "names a folder"},
{"nope", "no entry"},
} {
@@ -134,8 +134,53 @@ func TestRecordSQLInsert(t *testing.T) {
t.Fatal(err)
}
got := r.SQLInsert("people")
if got != "INSERT INTO people (last) VALUES ('O''Brien');" {
t.Fatalf("SQLInsert() = %q, want the single quote doubled", got)
if got != `INSERT INTO "people" ("last") VALUES ('O''Brien');` {
t.Fatalf("SQLInsert() = %q, want quoted identifiers and the single quote doubled", got)
}
}
func TestRecordSharesAReferenceAcrossColumns(t *testing.T) {
dir := writeData(t, map[string]string{
"currency": `[{"format":"{code}","code":"AUD","symbol":"$"},{"format":"{code}","code":"EUR","symbol":"€"}]`,
"price": `{"format":"","code":"{/currency.code}","symbol":"{/currency.symbol}"}`,
})
f := newGenerator(t, dir, WithSeed(1))
for i := 0; i < 100; i++ {
r, err := f.Record("price")
if err != nil {
t.Fatal(err)
}
m := map[string]string{}
for _, c := range r.Fields() {
m[c.Name] = c.Value
}
switch m["code"] {
case "AUD":
if m["symbol"] != "$" {
t.Fatalf("record %q: code AUD but symbol %q, want one currency draw across columns", r.JSON(), m["symbol"])
}
case "EUR":
if m["symbol"] != "€" {
t.Fatalf("record %q: code EUR but symbol %q, want one currency draw across columns", r.JSON(), m["symbol"])
}
default:
t.Fatalf("record %q has unexpected code %q", r.JSON(), m["code"])
}
}
}
func TestRecordSQLQuotesIdentifiers(t *testing.T) {
dir := writeData(t, map[string]string{
"row": `{"format": "", "postal-code": "1", "street-number": "2"}`,
})
f := newGenerator(t, dir, WithSeed(1))
r, err := f.Record("row")
if err != nil {
t.Fatal(err)
}
got := r.SQLInsert("my-table")
if got != `INSERT INTO "my-table" ("postal-code", "street-number") VALUES ('1', '2');` {
t.Fatalf("SQLInsert() = %q, want hyphenated identifiers and table quoted", got)
}
}
+15 -7
View File
@@ -52,15 +52,22 @@ func descend(s *session, root node, segments []string) (node, error) {
// render evaluates a compiled node to a string. compile validates every node up
// front, so rendering a compiled tree cannot fail.
func render(s *session, n node) string {
return renderShared(s, n, nil)
}
// renderShared is render with a shared draw context: the draws a record shares
// across its columns. A nil shared means a standalone render, where a reference
// is drawn per expansion as it always has been.
func renderShared(s *session, n node, shared *draws) string {
switch n := n.(type) {
case *choice:
return render(s, pick(s, n))
return renderShared(s, pick(s, n), shared)
case *template:
if n.repeat == 1 {
if n.fixed {
return n.lit
}
return expand(s, n)
return expand(s, n, shared)
}
var b strings.Builder
b.Grow(n.repeat * (n.grow + len(n.separator)))
@@ -68,7 +75,7 @@ func render(s *session, n node) string {
if i > 0 {
b.WriteString(n.separator)
}
b.WriteString(expand(s, n))
b.WriteString(expand(s, n, shared))
}
return b.String()
default:
@@ -90,11 +97,12 @@ func pick(r rng, c *choice) node {
// expand renders a template's compiled ops. compile validated every token, so this
// cannot fail.
func expand(s *session, t *template) string {
func expand(s *session, t *template, shared *draws) string {
var b strings.Builder
b.Grow(t.grow)
// One draw per held name, for this expansion only: a nested template and each
// repeat iteration get their own, since each is its own expansion.
// repeat iteration get their own, since each is its own expansion. A shared
// draw context, when a record supplies one, overrides that for references.
var held *draws
if len(t.held) > 0 {
held = &draws{
@@ -108,7 +116,7 @@ func expand(s *session, t *template) string {
case 'l':
b.WriteString(o.lit)
case 'f':
b.WriteString(readField(s, t, held, o.arms[s.IntN(len(o.arms))]))
b.WriteString(readField(s, t, held, shared, o.arms[s.IntN(len(o.arms))]))
case 'b':
// Read before the call, so the value a calc computes is the value the
// format showed. calcVars fixed the order op.operands holds.
@@ -116,7 +124,7 @@ func expand(s *session, t *template) string {
if len(o.operands) > 0 {
operands = make([]string, len(o.operands))
for j, a := range o.operands {
operands[j] = readField(s, t, held, a)
operands[j] = readField(s, t, held, shared, a)
}
}
b.WriteString(o.call(s, b.String(), operands)) // b.String() is the output so far
+1 -1
View File
@@ -315,7 +315,7 @@ func TestGrowIsALowerBound(t *testing.T) {
t.Fatalf("format %q did not compile to a template", format)
}
for i := 0; i < 50; i++ {
if got := len(expand(f.rand, tmpl)); got < tmpl.grow {
if got := len(expand(f.rand, tmpl, nil)); got < tmpl.grow {
t.Errorf("format %q: expand emitted %d bytes, below grow %d", format, got, tmpl.grow)
}
}