Frame json as an array and add ndjson

README.md

@@ -41,7 +41,7 @@ brace, a bracket or a quote, so the two cannot collide (see
 | `-s`, `--seed N` | reproducible output |
 | `-n`, `--repeat N` | render the value N times (up to 1048576), each an independent draw, streamed |
 | `--separator S` | between repeated values (default a newline) |
-| `--format F` | `text` (default), `json`, `csv` or `sql` — a record's columns, one record per row |
+| `--format F` | `text` (default), `json`, `ndjson`, `csv` or `sql` — a record's columns, one record per row (json frames them as an array) |
 | `--table T` | the INSERT target for `--format sql` (default: the path's last segment, or `records` for an inline template) |
 | `--list` | print every path, then exit |
 | `--version`, `-h`, `--help` | print, then exit |
@@ -81,7 +81,7 @@ For structured output a record writes the row for you.
 ### Records
 
 A record is a template seen as columns: its fields are the columns, its `format`
-the whole. `--format json|csv|sql` streams one record per row; the library's
+the whole. `--format json|ndjson|csv|sql` writes the records; the library's
 `Record` (below) hands back the columns. Every column is a string — this is the
 out-of-scope of typed scalars, see [Decisions](#decisions). Save
 `mydata/users.json`:
@@ -95,17 +95,17 @@ out-of-scope of typed scalars, see [Decisions](#decisions). Save
 ```
 
 ```sh
-fejkdata --seed 1 --data-path ./mydata --format json users   # {"first":"Bo","last":"Lovelace"}
+fejkdata --seed 1 --data-path ./mydata --format json users            # [{"first":"Bo","last":"Lovelace"}]
+fejkdata --seed 1 --data-path ./mydata --format ndjson users          # {"first":"Bo","last":"Lovelace"}
 fejkdata --seed 1 --data-path ./mydata --format csv  users            # first,last  →  Bo,Lovelace
 fejkdata --seed 1 --data-path ./mydata --format sql  users            # INSERT INTO "users" ("first", "last") VALUES ('Bo', 'Lovelace');
-fejkdata --seed 1 --data-path ./mydata --format json --repeat 3 users   # three objects, one per line
 fejkdata --seed 1 --data-path ./mydata --format sql --table people users # INSERT into another table
 ```
 
-`--repeat` streams that many records — newline-delimited JSON (one object per
-line, NDJSON), a CSV row after a header, or an INSERT in SQL. To fold NDJSON into
-a single array, `fejkdata … --format json | jq -s .`. Only a category-level
-template is a record; a field, choice or folder errors, and so does a `repeat` on
+`--repeat` streams that many records — `json` frames them as one array document,
+`ndjson` writes one object per line, `csv` a row after a header, `sql` one INSERT
+per line. Only a category-level template is a record; a field, choice or folder
+errors, and so does a `repeat` on
 the template itself, which composes the format into one string rather than
 projecting columns — ask for more records with `--repeat`. A `repeat` on a column
 is fine.
@@ -165,9 +165,9 @@ r, err = f.FakeRecord(`{"format":"{x}","x":["a","b"]}`) // compile + render inli
 | `WithoutShippedData()` | load only what you give |
 
 A `*Record` carries its columns via `Columns()`, and serializes them with `JSON()`
-(one object), `CSVHeader()`/`CSVLine()`, or `SQLInsert(table)` — the same three
-shapes the CLI's `--format` streams. `Record` and `FakeRecord` take a record; a
-path or template that is not one — a bare string, a choice, or a folder — errors.
+(one object), `CSVHeader()`/`CSVLine()`, or `SQLInsert(table)` — the shapes the
+CLI's `--format` writes. `Record` and `FakeRecord` take a record; a path or
+template that is not one — a bare string, a choice, or a folder — errors.
 
 A `*Generator` is safe for concurrent use; a seeded sequence is reproducible only
 when drawn from one goroutine. Changing how a value is composed shifts the seeded

cmd/fejkdata/main.go

@@ -35,12 +35,12 @@ or a quote). Templates reach the data by reference from the root —
 argument carrying a bracket, a closing brace or a quote but no valid JSON names
 neither.
 
-With --format json, csv or sql the argument must name a record — a template whose
-fields are its columns — and each render is streamed as one JSON object, one CSV
-row (after a header), or one INSERT.
+With --format json, ndjson, csv or sql the argument must name a record — a
+template whose fields are its columns — and the rows are written as one JSON
+array, one JSON object per line, one CSV row (after a header), or one INSERT.
 
   -d, --data-path D      a data directory to layer over the shipped data (repeatable; last wins on a clash)
-      --format F         output form: text (default), json, csv or sql
+      --format F         output form: text (default), json, ndjson, csv or sql
   -h, --help             print this help, then exit
       --list             list the paths the data offers, then exit
       --no-shipped-data  load only the --data-path directories
@@ -209,20 +209,28 @@ func parseArgs(argv []string) (invocation, error) {
 	return in, nil
 }
 
-// recordFormat is one way to write a record out: the line it renders, and the
-// header that precedes the first one, if the format has one.
+// recordFormat is one way to write a record out: the line each record renders,
+// the header that precedes the first one, and the open/close frame plus the
+// between-record separator a document form needs.
 type recordFormat struct {
 	header func(*fejkdata.Record) string
 	line   func(r *fejkdata.Record, table string) string
+	open   string
+	close  string
+	sep    string
 }
 
-// recordFormats is every --format that writes records.
+// recordFormats is every --format that writes records. json frames the records
+// as one array document; ndjson is the same column, one object per line.
 var recordFormats = map[string]recordFormat{
-	"csv":  {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }},
-	"json": {line: func(r *fejkdata.Record, _ string) string { return r.JSON() }},
-	"sql":  {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }},
+	"csv":    {header: (*fejkdata.Record).CSVHeader, line: func(r *fejkdata.Record, _ string) string { return r.CSVLine() }, sep: "\n"},
+	"json":   {line: jsonLine, open: "[", close: "]", sep: ",\n"},
+	"ndjson": {line: jsonLine, sep: "\n"},
+	"sql":    {line: func(r *fejkdata.Record, table string) string { return r.SQLInsert(table) }, sep: "\n"},
 }
 
+func jsonLine(r *fejkdata.Record, _ string) string { return r.JSON() }
+
 // writesRecords reports whether the format writes records rather than plain text.
 func (in invocation) writesRecords() bool {
 	return in.format != "text"
@@ -295,7 +303,10 @@ func (in invocation) options() []fejkdata.Option {
 // so a render failure comes before anything is written; a write failure surfaces
 // from Flush, bufio keeping the first one.
 func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) error {
-	draw, err := in.draw(f, kind, in.paths[0])
+	if in.writesRecords() {
+		return in.writeRecords(f, kind, w)
+	}
+	draw, err := in.textDraw(f, kind, in.paths[0])
 	if err != nil {
 		return err
 	}
@@ -314,10 +325,9 @@ func (in invocation) write(f *fejkdata.Generator, kind argKind, w io.Writer) err
 	return out.Flush()
 }
 
-// draw builds what one render yields: the value's text, or the record's line in
-// the chosen format, the header carried ahead of the first one.
-func (in invocation) draw(f *fejkdata.Generator, kind argKind, arg string) (func() (string, error), error) {
-	if !in.writesRecords() {
+// textDraw builds what one text render yields: the value, from a path or an
+// inline template.
+func (in invocation) textDraw(f *fejkdata.Generator, kind argKind, arg string) (func() (string, error), error) {
 	if kind != argTemplate {
 		return func() (string, error) { return f.Fake(arg) }, nil
 	}
@@ -326,31 +336,59 @@ func (in invocation) draw(f *fejkdata.Generator, kind argKind, arg string) (func
 		return nil, templateError{err}
 	}
 	return func() (string, error) { return t.Fake(), nil }, nil
-	}
+}
+
+// recordStream builds the record drawer for the argument, plus the INSERT table
+// a sql format names.
+func (in invocation) recordStream(f *fejkdata.Generator, kind argKind, arg string) (func() (*fejkdata.Record, error), string, error) {
 	record := func() (*fejkdata.Record, error) { return f.Record(arg) }
 	if kind == argTemplate {
 		t, err := f.NewRecordTemplate(arg)
 		if err != nil {
-			return nil, templateError{err}
+			return nil, "", templateError{err}
 		}
 		record = func() (*fejkdata.Record, error) { return t.Fake(), nil }
 	}
-	format, table, first := recordFormats[in.format], in.table, true
+	table := in.table
 	if table == "" {
 		table = defaultTable(arg, kind)
 	}
-	return func() (string, error) {
+	return record, table, nil
+}
+
+// writeRecords streams a record per line in the chosen format, framing a document
+// form with its open/close brackets and a header preceding the first record.
+func (in invocation) writeRecords(f *fejkdata.Generator, kind argKind, w io.Writer) error {
+	record, table, err := in.recordStream(f, kind, in.paths[0])
+	if err != nil {
+		return err
+	}
+	format := recordFormats[in.format]
+	out := bufio.NewWriter(w)
+	if format.open != "" {
+		out.WriteString(format.open)
+		out.WriteByte('\n')
+	}
+	for i := 0; i < in.repeat; i++ {
 		r, err := record()
 		if err != nil {
-			return "", err
+			return err
 		}
-		line := format.line(r, table)
-		if first && format.header != nil {
-			line = format.header(r) + "\n" + line
+		if i == 0 && format.header != nil {
+			out.WriteString(format.header(r))
+			out.WriteByte('\n')
 		}
-		first = false
-		return line, nil
-	}, nil
+		if i > 0 {
+			out.WriteString(format.sep)
+		}
+		out.WriteString(format.line(r, table))
+	}
+	if format.close != "" {
+		out.WriteByte('\n')
+		out.WriteString(format.close)
+	}
+	out.WriteByte('\n')
+	return out.Flush()
 }
 
 // defaultTable names the INSERT target when --table is absent: the path's last

cmd/fejkdata/main_test.go

@@ -463,16 +463,33 @@ func recordDir(t *testing.T) string {
 }
 
 func TestRunRecordJSON(t *testing.T) {
-	code, out, errb := runOut("--seed", "1", "--format", "json", "--data-path", recordDir(t), "users")
+	code, out, errb := runOut("--seed", "1", "--format", "json", "--repeat", "2", "--data-path", recordDir(t), "users")
 	if code != 0 {
 		t.Fatalf("run = %d, stderr=%q", code, errb)
 	}
-	var m map[string]string
-	if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &m); err != nil {
-		t.Fatalf("json output is not one valid JSON object: %v\n%q", err, out)
+	var arr []map[string]string
+	if err := json.Unmarshal([]byte(out), &arr); err != nil {
+		t.Fatalf("json output is not one JSON array: %v\n%q", err, out)
+	}
+	if len(arr) != 2 || len(arr[0]) != 2 || arr[0]["first"] == "" || arr[0]["last"] == "" {
+		t.Fatalf("json output = %q, want an array of two records with first and last columns", out)
+	}
+}
+
+func TestRunRecordNDJSON(t *testing.T) {
+	code, out, errb := runOut("--seed", "1", "--format", "ndjson", "--repeat", "2", "--data-path", recordDir(t), "users")
+	if code != 0 {
+		t.Fatalf("run = %d, stderr=%q", code, errb)
+	}
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	if len(lines) != 2 {
+		t.Fatalf("ndjson output = %d lines %q, want one object per line", len(lines), out)
+	}
+	for _, line := range lines {
+		var m map[string]string
+		if err := json.Unmarshal([]byte(line), &m); err != nil || len(m) != 2 {
+			t.Fatalf("ndjson line %q is not one object: %v", line, err)
 		}
-	if len(m) != 2 || m["first"] == "" || m["last"] == "" {
-		t.Fatalf("json output = %q, want first and last columns", out)
 	}
 }
 
@@ -520,9 +537,9 @@ func TestRunRecordInlineTemplate(t *testing.T) {
 	if code != 0 {
 		t.Fatalf("inline record = %d, stderr=%q", code, errb)
 	}
-	var m map[string]string
-	if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &m); err != nil || m["x"] != "a" && m["x"] != "b" {
-		t.Fatalf("inline record json = %q, want a column x (err %v)", out, err)
+	var arr []map[string]string
+	if err := json.Unmarshal([]byte(out), &arr); err != nil || len(arr) != 1 || (arr[0]["x"] != "a" && arr[0]["x"] != "b") {
+		t.Fatalf("inline record json = %q, want one record with a column x (err %v)", out, err)
 	}
 }
 
@@ -531,7 +548,7 @@ func TestRunRecordMisuse(t *testing.T) {
 		args []string
 		want string
 	}{
-		{[]string{"--format", "yaml", "users"}, "--format takes csv, json, sql or text"},
+		{[]string{"--format", "yaml", "users"}, "--format takes csv, json, ndjson, sql or text"},
 		{[]string{"--format", "json", "--separator", ",", "users"}, "--separator joins text values"},
 		{[]string{"--table", "t", "users"}, "--table names the INSERT target"},
 		{[]string{"--format", "json", "--table", "t", "users"}, "--table names the INSERT target"},