larvit/fejkdata
3
0
Fork 0
Code Pull Requests 1 Actions Activity

Compile format strings at New instead of re-scanning them on every render

Browse Source
This commit is contained in:
lilleman
2026-08-27 23:09:35 +02:00
committed by lilleman-tw
parent 5282f37a59
commit ab9846c2ff
8 changed files with 202 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1
View File
@@ -391,6 +391,7 @@ Source is bind-mounted; build caches persist in the `gocache` volume.
docker compose run --rm test # run tests
docker compose run --rm ci # vet + format check + tests
docker compose run --rm cover # tests with coverage
docker compose run --rm bench # benchmarks
docker compose run --rm build # compile the library
docker compose run --rm vet # go vet
docker compose run --rm dev # interactive shell
bench_test.go
+72
View File
@@ -0,0 +1,72 @@
package fakes
import (
"os"
"path/filepath"
"testing"
)
func benchFaker(b *testing.B, dir string) *Fakes {
b.Helper()
f, err := New([]string{dir}, WithSeed(1))
if err != nil {
b.Fatal(err)
}
return f
}
func benchPath(b *testing.B, dir, path string) {
f := benchFaker(b, dir)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := f.Fake(path); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkPerson(b *testing.B) { benchPath(b, "data/sv_SE", "person") }
func BenchmarkAddress(b *testing.B) { benchPath(b, "data/sv_SE", "address") }
func BenchmarkWord(b *testing.B) { benchPath(b, "data/sv_SE", "word") }
func BenchmarkCreditcard(b *testing.B) { benchPath(b, "data/misc", "creditcard") }
func BenchmarkSSN(b *testing.B) { benchPath(b, "data/sv_SE", "ssn") }
func BenchmarkUUIDv7(b *testing.B) { benchPath(b, "data/misc", "uuid") }
// tmpData writes one category JSON into a fresh dir and returns the dir.
func tmpData(b *testing.B, name, body string) string {
b.Helper()
dir := b.TempDir()
if err := os.WriteFile(filepath.Join(dir, name+".json"), []byte(body), 0o644); err != nil {
b.Fatal(err)
}
return dir
}
// BenchmarkCalc exercises the per-render expression re-parse.
func BenchmarkCalc(b *testing.B) {
dir := tmpData(b, "inv", `{"format":"{net} x {qty} = {calc(net * qty, 2)}","net":["19.99"],"qty":["3"]}`)
benchPath(b, dir, "inv")
}
// BenchmarkLongLiteral exercises the per-rune literal write path.
func BenchmarkLongLiteral(b *testing.B) {
dir := tmpData(b, "sql", `{"format":"INSERT INTO customers (id, name, city) V#ALUES (#1#2#3, '{word}', '{word}');","word":["alpha","beta","gamma","delta"]}`)
benchPath(b, dir, "sql")
}
// BenchmarkRepeat exercises repeat + separator.
func BenchmarkRepeat(b *testing.B) {
dir := tmpData(b, "many", `{"format":"{word}","repeat":20,"separator":", ","word":["alpha","beta","gamma","delta"]}`)
benchPath(b, dir, "many")
}
// BenchmarkNew measures load+compile+validate of the whole shipped tree.
func BenchmarkNew(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if _, err := New([]string{"data"}, WithSeed(1)); err != nil {
b.Fatal(err)
}
}
}
builtins.go
+28 -15
View File
@@ -32,28 +32,47 @@ var builtins = map[string]builtin{
"uuid": {arity: 0, call: func(s *session, _ string, _ map[string]node, _ []string) string { return uuidV7(s) }},
"ulid": {arity: 0, call: func(s *session, _ string, _ map[string]node, _ []string) string { return ulid(s) }},
"objectid": {arity: 0, call: func(s *session, _ string, _ map[string]node, _ []string) string { return randHex(s, 24) }},
"nanoid": {arity: 1, check: posIntArg, call: func(s *session, _ string, _ map[string]node, a []string) string { return nanoid(s, atoi(a[0])) }},
"hex": {arity: 1, check: posIntArg, call: func(s *session, _ string, _ map[string]node, a []string) string { return randHex(s, atoi(a[0])) }},
"base64": {arity: 1, check: posIntArg, call: func(s *session, _ string, _ map[string]node, a []string) string {
return base64.StdEncoding.EncodeToString(randBytes(s, atoi(a[0])))
"nanoid": {arity: 1, check: posIntArg, prep: func(a []string) callFn {
n := atoi(a[0])
return func(s *session, _ string, _ map[string]node) string { return nanoid(s, n) }
}},
"int": {arity: 2, check: intRangeArgs, call: func(s *session, _ string, _ map[string]node, a []string) string {
return strconv.Itoa(atoi(a[0]) + s.IntN(atoi(a[1])-atoi(a[0])+1))
"hex": {arity: 1, check: posIntArg, prep: func(a []string) callFn {
n := atoi(a[0])
return func(s *session, _ string, _ map[string]node) string { return randHex(s, n) }
}},
"base64": {arity: 1, check: posIntArg, prep: func(a []string) callFn {
n := atoi(a[0])
return func(s *session, _ string, _ map[string]node) string {
return base64.StdEncoding.EncodeToString(randBytes(s, n))
}
}},
"int": {arity: 2, check: intRangeArgs, prep: func(a []string) callFn {
lo, span := atoi(a[0]), atoi(a[1])-atoi(a[0])+1
return func(s *session, _ string, _ map[string]node) string { return strconv.Itoa(lo + s.IntN(span)) }
}},
"float": {arity: 3, check: floatArgs, prep: func(a []string) callFn {
lo, _ := strconv.ParseFloat(a[0], 64)
hi, _ := strconv.ParseFloat(a[1], 64)
dp := atoi(a[2])
return func(s *session, _ string, _ map[string]node) string {
return strconv.FormatFloat(lo+s.Float64()*(hi-lo), 'f', dp, 64)
}
}},
"float": {arity: 3, check: floatArgs, call: floatCall},
"iban": {arity: 1, check: ibanArg, call: func(s *session, _ string, _ map[string]node, a []string) string { return iban(s, a[0]) }},
// calc is the one builtin that reads the sibling fields (to render its operands);
// every other ignores them. 1 or 2 args: the expression and an optional decimals.
"calc": {arity: -1, check: checkCalc, call: calcCall},
"calc": {arity: -1, check: checkCalc, prep: calcPrep},
// seq is the one stateful builtin: a per-session counter from 1, advancing on
// each call. An optional name selects an independent counter; no name uses the
// default one. Deterministic by construction, so a seeded faker stays stable.
"seq": {arity: -1, check: seqArg, call: func(s *session, _ string, _ map[string]node, a []string) string {
"seq": {arity: -1, check: seqArg, prep: func(a []string) callFn {
key := ""
if len(a) == 1 {
key = a[0]
}
return func(s *session, _ string, _ map[string]node) string {
return strconv.FormatUint(s.next(key), 10)
}
}},
}
@@ -123,12 +142,6 @@ func floatArgs(_ map[string]node, a []string) error {
return nil
}
func floatCall(s *session, _ string, _ map[string]node, a []string) string {
lo, _ := strconv.ParseFloat(a[0], 64)
hi, _ := strconv.ParseFloat(a[1], 64)
return strconv.FormatFloat(lo+s.Float64()*(hi-lo), 'f', atoi(a[2]), 64)
}
func seqArg(_ map[string]node, a []string) error {
if len(a) > 1 {
return fmt.Errorf("seq takes at most one name, got %d args", len(a))
calc.go
+5 -5
View File
@@ -82,17 +82,17 @@ func checkCalc(fields map[string]node, args []string) error {
return nil
}
// calcCall renders a calc token. checkCalc proved the expression parses and the
// decimals arg is valid, so neither step here can fail. dp -1 prints the minimal
// form; a given dp rounds to that many places. The emitted-so-far arg is unused —
// calc reads sibling fields, not the preceding output.
func calcCall(s *session, _ string, fields map[string]node, args []string) string {
// calcPrep parses the expression and decimals once, at compile time. checkCalc
// proved both valid, so neither step here can fail; dp -1 prints the minimal form.
func calcPrep(args []string) callFn {
expr, _ := parseCalc(args[0])
dp := -1
if len(args) == 2 {
dp = atoi(args[1])
}
return func(s *session, _ string, fields map[string]node) string {
return strconv.FormatFloat(expr.eval(s, fields), 'f', dp, 64)
}
}
// calcOperands lists the sibling-field names every {calc(...)} token in a format
compose.yaml
+4
View File
@@ -32,6 +32,10 @@ services:
<<: *go
command: go test -cover ./...
bench:
<<: *go
command: go test -run XXX -bench . -benchmem ./...
vet:
<<: *go
command: go vet ./...
node.go
+3
View File
@@ -43,6 +43,8 @@ type template struct {
fields map[string]node
repeat int
separator string
ops []op // format compiled once (see compileOps); what expand walks
grow int // minimum output size, to size the render buffer
}
func (*template) isNode() {}
@@ -136,6 +138,7 @@ func compileTemplate(m map[string]any) (node, error) {
if err := checkTokens(format, t.fields); err != nil {
return nil, err
}
t.ops, t.grow = compileOps(format, t.fields)
return t, nil
}
render.go
+19 -22
View File
@@ -89,14 +89,15 @@ func render(s *session, n node) string {
return render(s, pick(s, n))
case *template:
if n.repeat == 1 {
return expand(s, n.format, n.fields)
return expand(s, n)
}
var b strings.Builder
b.Grow(n.repeat * (n.grow + len(n.separator)))
for i := 0; i < n.repeat; i++ {
if i > 0 {
b.WriteString(n.separator)
}
b.WriteString(expand(s, n.format, n.fields))
b.WriteString(expand(s, n))
}
return b.String()
default:
@@ -131,34 +132,30 @@ func classChar(s *session, c rune) byte {
}
}
// expand renders a format string: literals verbatim, class chars randomised, a
// {name(args)} token via its builtin, any other {token} via resolve. checkTokens
// validated every token at compile time, so the scan cannot fail here (the
// eachToken error is unreachable and ignored).
func expand(s *session, format string, fields map[string]node) string {
// expand renders a template's compiled ops. compile validated every token, so this
// cannot fail.
func expand(s *session, t *template) string {
var b strings.Builder
_ = eachToken(format, func(t ftoken) error {
switch t.kind {
b.Grow(t.grow)
for i := range t.ops {
o := &t.ops[i]
switch o.kind {
case 'l':
b.WriteRune(t.r)
b.WriteString(o.lit)
case 'c':
b.WriteByte(classChar(s, t.r))
b.WriteByte(classChar(s, o.r))
case 'f':
b.WriteString(resolve(s, o.names, t.fields))
case 'b':
if name, args, ok := funcCall(t.body); ok {
b.WriteString(builtins[name].call(s, b.String(), fields, args)) // b.String() is the output so far
} else {
b.WriteString(resolve(s, t.body, fields))
b.WriteString(o.call(s, b.String(), t.fields)) // b.String() is the output so far
}
}
return nil
})
return b.String()
}
// resolve renders a "{token}" body: one or more names separated by '|', one
// picked at random. A name is a sibling field or a {..path} reference, which
// linkRefs bound into fields too; checkTokens and linkRefs guarantee both exist.
func resolve(s *session, token string, fields map[string]node) string {
names := strings.Split(token, "|")
// resolve renders one field alternation: the '|' arms, one picked at random. A
// name is a sibling field or a {..path} reference, which linkRefs bound into
// fields too; checkTokens and linkRefs guarantee both exist.
func resolve(s *session, names []string, fields map[string]node) string {
return render(s, fields[names[s.IntN(len(names))]])
}
template.go
+66
View File
@@ -69,6 +69,8 @@ func eachToken(format string, fn func(ftoken) error) error {
// compile time (their values, beyond the count). The registry lives in builtins.go.
type builtin struct {
arity int
// prep parses validated args once, at compile time, into the closure expand calls.
prep func(args []string) callFn
check func(fields map[string]node, args []string) error
call func(s *session, emitted string, fields map[string]node, args []string) string
}
@@ -159,3 +161,67 @@ func fieldTokens(format string) []string {
})
return names
}
// op is one compiled unit of a format string: a literal run, a class char, a field
// alternation, or a builtin already bound to its args. compile builds these so
// render never re-scans the format.
// callFn is a builtin bound to one call site: its args already parsed.
type callFn func(s *session, emitted string, fields map[string]node) string
type op struct {
kind byte // 'l' literal run, 'c' class char, 'f' field alternation, 'b' builtin
lit string // kind 'l'
r rune // kind 'c'
names []string // kind 'f': the '|' arms, split once
call callFn
}
// compileOps turns a validated format string into ops, and returns the smallest
// output it can produce (literals plus one byte per class char) to size the buffer.
func compileOps(format string, fields map[string]node) ([]op, int) {
var ops []op
var lit strings.Builder
grow := 0
flush := func() {
if lit.Len() > 0 {
ops = append(ops, op{kind: 'l', lit: lit.String()})
lit.Reset()
}
}
_ = eachToken(format, func(t ftoken) error {
switch t.kind {
case 'l':
lit.WriteRune(t.r)
case 'c':
flush()
grow++
ops = append(ops, op{kind: 'c', r: t.r})
case 'b':
flush()
if name, args, ok := funcCall(t.body); ok {
ops = append(ops, op{kind: 'b', call: builtins[name].bind(args)})
} else {
ops = append(ops, op{kind: 'f', names: strings.Split(t.body, "|")})
}
}
return nil
})
flush()
for _, o := range ops {
if o.kind == 'l' {
grow += len(o.lit)
}
}
return ops, grow
}
// bind returns the closure for one call site, parsing args once via prep if present.
func (b builtin) bind(args []string) callFn {
if b.prep != nil {
return b.prep(args)
}
call := b.call
return func(s *session, emitted string, fields map[string]node) string {
return call(s, emitted, fields, args)
}
}