Make great documentation goal 1.2, and plan the README's reorganisation and the nested-template gaps #33

Open
lilleman wants to merge 966 commits from worktree-todo-relative-dates-exact-counts into main
9 changed files with 186 additions and 208 deletions
Showing only changes of commit 467cb6fb18 - Show all commits
+8 -16
View File
@@ -127,31 +127,23 @@ func renderEdges(n node) []renderEdge {
return es
case *template:
var es []renderEdge
add := func(name, operand string) {
a := splitArm(name, n.refs)
add := func(a arm, operand string) {
c := n.head(a.key)
if c == nil {
return
}
for _, leaf := range pathLeaves(c, a.tail) {
es = append(es, renderEdge{leaf, name, operand})
es = append(es, renderEdge{leaf, a.name, operand})
}
}
_ = eachToken(n.format, func(t ftoken) error {
if t.kind != 'b' {
return nil
for _, o := range n.ops {
for _, a := range o.operands {
add(a, o.fn)
}
if fn, _, isFunc := funcCall(t.body); isFunc {
for _, operand := range tokenOperands(t.body) {
add(operand, fn)
}
return nil
for _, a := range o.arms {
add(a, "")
}
for _, name := range splitOutside(t.body, '|') {
add(name, "")
}
return nil
})
}
return es
case *table:
return []renderEdge{{to: n.format, label: "format"}}
+21 -42
View File
@@ -15,7 +15,7 @@ func heldCheck(path string, n node) error {
if !ok || len(t.held) == 0 {
return nil
}
readers := boundReaders(t.format, t.bound, t.refs)
readers := boundReaders(t.ops, t.bound)
for _, head := range heldHeads(t) {
if _, isPath := t.bound[head]; isPath && isRef(head) {
continue
@@ -87,21 +87,14 @@ func checkHeadHeld(t *template, head string, readers []reader) error {
// operandReader names the builtin whose operand holds head.
func operandReader(t *template, head string) string {
fn := ""
_ = eachToken(t.format, func(tok ftoken) error {
if tok.kind != 'b' || fn != "" {
return nil
}
if name, _, isFunc := funcCall(tok.body); isFunc {
for _, operand := range tokenOperands(tok.body) {
if splitArm(operand, t.refs).key == head {
fn = name
}
for _, o := range t.ops {
for _, a := range o.operands {
if a.key == head {
return o.fn
}
}
return nil
})
return fn
}
return ""
}
// coverPath collects what holding one path pins: every choice level the path
@@ -183,8 +176,8 @@ func renders(n node, want, seen map[node]bool) bool {
// disagree. Reads are compared in sorted order, so which pair is reported does not
// depend on where the tokens sit.
// docs/decisions.md#a-bare-reference-draws-each-time-a-reference-path-is-held
func checkNoOverlap(format string, bound map[string]string, refs map[string]refBinding) error {
names := boundReaders(format, bound, refs)
func checkNoOverlap(ops []op, bound map[string]string) error {
names := boundReaders(ops, bound)
// Stable over one format-order scan, so two readers of one name (a token and a
// calc operand both naming "p") are reported as the format writes them.
sort.SliceStable(names, func(i, j int) bool { return names[i].path < names[j].path })
@@ -205,28 +198,20 @@ type reader struct{ name, path, label string }
// boundReaders lists every way a format reaches a bound sibling field, in the order the
// format writes them. An operand renders its field, so it names a level exactly
// as a token does; one scan finds both, which is what puts them in one order.
func boundReaders(format string, bound map[string]string, refs map[string]refBinding) []reader {
func boundReaders(ops []op, bound map[string]string) []reader {
var names []reader
_ = eachToken(format, func(t ftoken) error {
if t.kind != 'b' {
return nil
}
if fn, _, isFunc := funcCall(t.body); isFunc {
for _, operand := range tokenOperands(t.body) {
a := splitArm(operand, refs)
if _, isBound := bound[a.key]; isBound && !isRef(a.key) {
names = append(names, reader{a.name, a.path, fmt.Sprintf("%s operand %q", fn, operand)})
}
for _, o := range ops {
for _, a := range o.operands {
if _, isBound := bound[a.key]; isBound && !isRef(a.key) {
names = append(names, reader{a.name, a.path, fmt.Sprintf("%s operand %q", o.fn, a.name)})
}
return nil
}
for _, a := range splitArms(t.body, refs) {
for _, a := range o.arms {
if _, isBound := bound[a.key]; isBound && !isRef(a.key) {
names = append(names, reader{a.name, a.path, "token {" + a.name + "}"})
}
}
return nil
})
}
return names
}
@@ -234,16 +219,10 @@ func boundReaders(format string, bound map[string]string, refs map[string]refBin
// {uppercase(w)} would read one draw twice, where {w} {w} alone draws twice. The
// error names the single-token spelling.
// docs/decisions.md#a-bare-reference-draws-each-time-a-reference-path-is-held
func checkNoRepeatedRead(format string, c formatOps, refs map[string]refBinding) error {
func checkNoRepeatedRead(c formatOps) error {
count := map[string]int{}
return eachToken(format, func(t ftoken) error {
if t.kind != 'b' {
return nil
}
if _, _, isFunc := funcCall(t.body); isFunc {
return nil
}
for _, a := range splitArms(t.body, refs) {
for _, o := range c.ops {
for _, a := range o.arms {
if len(a.tail) > 0 || !c.held[a.key] {
continue
}
@@ -251,6 +230,6 @@ func checkNoRepeatedRead(format string, c formatOps, refs map[string]refBinding)
return fmt.Errorf("token {%s} is repeated, and %s holds %q to one draw per expansion; write {%s} once", a.name, c.holder[a.key], a.key, a.name)
}
}
return nil
})
}
return nil
}
+10 -3
View File
@@ -107,8 +107,15 @@ func loneReference(arg string) (string, bool) {
raw = m["format"]
}
format, isString := raw.(string)
body, lone := loneRef(format)
if !isString || !lone {
if !isString {
return "", false
}
toks, err := parseFormat(format)
if err != nil {
return "", false
}
body, lone := loneRef(toks)
if !lone {
return "", false
}
if strings.HasPrefix(body, "/") {
@@ -161,7 +168,7 @@ func linkNodeRefs(scope nodeScope, root map[string]node) error {
return nil
}
t.keyDrawGroup("")
for _, name := range refTokens(t.format) {
for _, name := range refTokens(t.tokens) {
sigil, rest, err := refShape(name)
if err != nil {
return fmt.Errorf("%s: reference {%s}: %w", path, name, err)
+17 -8
View File
@@ -45,6 +45,7 @@ func (*null) isNode() {}
type template struct {
// Filled by `compileString`, `compileTemplate` and `table.compileWhole`:
format string
tokens []formatToken
fields map[string]node
repeat int
separator string
@@ -203,10 +204,14 @@ func jsonKind(v any) string {
}
func compileString(s string) (node, error) {
if err := checkTokens(s, nil); err != nil {
toks, err := parseFormat(s)
if err != nil {
return nil, err
}
t := &template{format: s, repeat: 1, fromString: true}
if err := checkTokens(toks, nil); err != nil {
return nil, err
}
t := &template{format: s, tokens: toks, repeat: 1, fromString: true}
if err := t.compileRefFree(); err != nil {
return nil, err
}
@@ -214,7 +219,7 @@ func compileString(s string) (node, error) {
}
func (t *template) compileRefFree() error {
if len(refTokens(t.format)) > 0 {
if len(refTokens(t.tokens)) > 0 {
return nil
}
return t.compileFormat()
@@ -223,7 +228,7 @@ func (t *template) compileRefFree() error {
// compileFormat compiles the format into ops, and applies the fences that need the
// compiled reads.
func (t *template) compileFormat() error {
c := compileOps(t.format, t.refs)
c := compileOps(t.tokens, t.refs)
t.ops, t.grow, t.bound, t.held, t.heldLocal = c.ops, c.grow, c.bound, c.held, c.heldLocal
t.fixed = true
for _, o := range t.ops {
@@ -234,10 +239,10 @@ func (t *template) compileFormat() error {
if t.fixed && len(t.ops) == 1 {
t.lit = t.ops[0].lit
}
if err := checkNoOverlap(t.format, t.bound, t.refs); err != nil {
if err := checkNoOverlap(t.ops, t.bound); err != nil {
return err
}
return checkNoRepeatedRead(t.format, c, t.refs)
return checkNoRepeatedRead(c)
}
func compileChoice(items []any, pos position) (node, error) {
@@ -331,13 +336,17 @@ func compileTemplate(m map[string]any, pos position) (node, error) {
if len(fields) == 0 && o.repeat == 1 && !o.weighted && o.datatype == DataTypeString && o.group == "" {
return nil, fmt.Errorf("an object holding only a format is a string; write %q", o.format)
}
if err := checkTokens(o.format, fields); err != nil {
toks, err := parseFormat(o.format)
if err != nil {
return nil, err
}
if err := checkTokens(toks, fields); err != nil {
return nil, err
}
if err := checkNestedDrawGroup(fields, o.group); err != nil {
return nil, err
}
t := &template{format: o.format, fields: fields, repeat: o.repeat, separator: o.separator, datatype: o.datatype, drawGroup: o.group, record: fieldPos == inColumn}
t := &template{format: o.format, tokens: toks, fields: fields, repeat: o.repeat, separator: o.separator, datatype: o.datatype, drawGroup: o.group, record: fieldPos == inColumn}
if err := t.compileRefFree(); err != nil {
return nil, err
}
+12 -16
View File
@@ -87,7 +87,7 @@ func linkRefs(root map[string]node) error {
// category it sits in.
// docs/decisions.md#a-category-never-references-itself-and-a-records-fences-run-at-load
func linkTemplateRefs(folder []string, path, category string, t *template, root map[string]node) error {
names := refTokens(t.format)
names := refTokens(t.tokens)
if len(names) == 0 {
return nil
}
@@ -130,7 +130,7 @@ type columnRead struct {
}
func columnReadOf(t *template) *columnRead {
name, lone := loneRef(t.format)
name, lone := loneRef(t.tokens)
if !lone || t.repeat != 1 {
return nil
}
@@ -143,17 +143,11 @@ func columnReadOf(t *template) *columnRead {
}
// loneRef is the reference a format of one reference token and nothing else reads.
func loneRef(format string) (string, bool) {
units, body := 0, ""
err := eachToken(format, func(t ftoken) error {
units++
body = t.body
return nil
})
if err != nil || units != 1 || !isRef(body) || strings.ContainsAny(body, "|(") {
func loneRef(toks []formatToken) (string, bool) {
if len(toks) != 1 || toks[0].kind != 'f' || len(toks[0].names) != 1 || !isRef(toks[0].names[0]) {
return "", false
}
return body, true
return toks[0].names[0], true
}
// eachTemplate calls fn once per template, with the folder its category sits in
@@ -220,13 +214,15 @@ func resolveCategory(root map[string]node, segments []string) (head []string, ta
}
// refTokens returns the reference names a format reads, as tokens or as operands.
func refTokens(format string) []string {
func refTokens(toks []formatToken) []string {
var refs []string
seen := map[string]bool{}
for _, name := range append(fieldTokens(format), operandTokens(format)...) {
if isRef(name) && !seen[name] {
seen[name] = true
refs = append(refs, name)
for _, tok := range toks {
for _, name := range tok.names {
if isRef(name) && !seen[name] {
seen[name] = true
refs = append(refs, name)
}
}
}
return refs
+14 -8
View File
@@ -324,16 +324,22 @@ func (t *table) checkCells() error {
// compileWhole compiles the format a row renders whole through, over the columns
// as its fields.
func (t *table) compileWhole(format string) error {
for _, name := range fieldTokens(format) {
a := splitArm(name, nil)
if _, ok := t.col[a.key]; !ok && !isRef(a.key) && a.key != "" {
return fmt.Errorf("format names no column %q of %s; the columns are %v", a.key, t.file, t.columns)
}
}
if err := checkTokens(format, t.fields); err != nil {
toks, err := parseFormat(format)
if err != nil {
return err
}
t.format = &template{format: format, fields: t.fields, repeat: 1, record: true, table: t}
for _, tok := range toks {
for _, name := range tok.names {
a := splitArm(name, nil)
if _, ok := t.col[a.key]; tok.kind == 'f' && !ok && !isRef(a.key) && a.key != "" {
return fmt.Errorf("format names no column %q of %s; the columns are %v", a.key, t.file, t.columns)
}
}
}
if err := checkTokens(toks, t.fields); err != nil {
return err
}
t.format = &template{format: format, tokens: toks, fields: t.fields, repeat: 1, record: true, table: t}
t.whole = &row{t}
return t.format.compileRefFree()
}
+100 -107
View File
@@ -57,6 +57,52 @@ func eachToken(format string, fn func(ftoken) error) error {
return nil
}
// formatToken is one parsed unit of a format: a literal run, a field alternation or a
// builtin call.
type formatToken struct {
kind byte // 'l' literal run, 'f' field alternation, 'b' builtin call
lit string // kind 'l'
body string // the braces' content, as written
fn string // kind 'b'
args []string // kind 'b'
names []string // kind 'f': the '|' arms; kind 'b': the operands its builtin reads
}
// parseFormat is the one reading of a format's tokens: a '(' outside a selector makes
// a token a call.
func parseFormat(format string) ([]formatToken, error) {
var toks []formatToken
var lit strings.Builder
flush := func() {
if lit.Len() > 0 {
toks = append(toks, formatToken{kind: 'l', lit: lit.String()})
lit.Reset()
}
}
err := eachToken(format, func(t ftoken) error {
if t.kind == 'l' {
lit.WriteRune(t.r)
return nil
}
flush()
if indexOutside(t.body, '(') < 0 {
toks = append(toks, formatToken{kind: 'f', body: t.body, names: splitOutside(t.body, '|')})
return nil
}
name, args, ok := funcCall(t.body)
if !ok {
return fmt.Errorf("malformed function token {%s}", t.body)
}
toks = append(toks, formatToken{kind: 'b', body: t.body, fn: name, args: args, names: builtinOperands(name, args)})
return nil
})
if err != nil {
return nil, err
}
flush()
return toks, nil
}
// builtin is a format-string function invoked as {name(args)}. It receives the
// session (its rng, and the {seq()} counters), the output emitted so far in the
// current expansion (for derivations such as a checksum over preceding digits), and
@@ -119,15 +165,11 @@ func plural(n int) string {
return "s"
}
// checkFunc validates a function token at compile time: well-formed, naming a
// known builtin, with the arg count that builtin takes and args its check accepts.
// fields is passed through for the builtins (calc, the transforms) that validate
// against them.
func checkFunc(body string, fields map[string]node) error {
name, args, ok := funcCall(body)
if !ok {
return fmt.Errorf("malformed function token {%s}", body)
}
// checkFunc validates a call at compile time: naming a known builtin, with the arg
// count that builtin takes and args its check accepts. fields is passed through for
// the builtins (calc, the transforms) that validate against them.
func checkFunc(tok formatToken, fields map[string]node) error {
body, name, args := tok.body, tok.fn, tok.args
b, known := builtins[name]
if !known {
return fmt.Errorf("token {%s}: unknown function %q", body, name)
@@ -143,34 +185,38 @@ func checkFunc(body string, fields map[string]node) error {
return nil
}
// checkTokens validates a format string the way expand scans it, so every
// "{token}" is balanced and names an existing field (or a known function). This
// makes a typo'd or dangling reference a New-time error, never a random
// render-time one.
func checkTokens(format string, fields map[string]node) error {
return eachToken(format, func(t ftoken) error {
if t.kind != 'b' {
return nil
// checkTokens proves every parsed token names an existing field or a known function,
// so a typo'd or dangling reference is a New-time error.
func checkTokens(toks []formatToken, fields map[string]node) error {
for _, t := range toks {
if err := checkToken(t, fields); err != nil {
return err
}
if indexOutside(t.body, '(') >= 0 { // a function token, not a field
return checkFunc(t.body, fields)
}
names := splitOutside(t.body, '|')
for _, name := range names {
}
return nil
}
func checkToken(t formatToken, fields map[string]node) error {
switch t.kind {
case 'b':
return checkFunc(t, fields)
case 'f':
for _, name := range t.names {
if isRef(name) {
if _, _, err := refShape(name); err != nil {
return fmt.Errorf("token {%s}: %w", t.body, err)
}
continue // its target is checked at New (see linkRefs)
}
if err := checkArm(name, fields, len(names) == 1); err != nil {
if err := checkArm(name, fields, len(t.names) == 1); err != nil {
return fmt.Errorf("token {%s}: %w", t.body, err)
}
}
// Last, so an arm broken on its own terms is reported as that: a repeat is
// the consequence of such a mistake, not the mistake itself.
return checkNoRepeatedArm(t.body, names)
})
return checkNoRepeatedArm(t.body, t.names)
}
return nil
}
// checkArm validates one sibling name or path against a template's fields.
@@ -210,32 +256,16 @@ func hintableRef(name string) bool {
return checkPathNames(name) == nil
}
// tokenOperands lists the fields one {token} body reads as operands, empty for a
// field token or a builtin that reads none.
func tokenOperands(body string) []string {
name, args, ok := funcCall(body)
if !ok {
return nil
}
// builtinOperands lists the fields a call reads as operands, empty for a builtin that
// reads none, or a call checkFunc refuses.
func builtinOperands(name string, args []string) []string {
b, known := builtins[name]
if !known || b.operands == nil {
if !known || b.operands == nil || (b.arity >= 0 && len(args) != b.arity) {
return nil
}
return b.operands(args)
}
// operandTokens lists every operand the builtins in a format read.
func operandTokens(format string) []string {
var names []string
_ = eachToken(format, func(t ftoken) error {
if t.kind == 'b' {
names = append(names, tokenOperands(t.body)...)
}
return nil
})
return names
}
// checkNoRepeatedArm rejects {a|a|b}: an alternation picks its arms evenly, so a
// repeated one is a second spelling of weight. The error names the spelling that
// does skew a pick.
@@ -253,20 +283,6 @@ func checkNoRepeatedArm(body string, names []string) error {
return nil
}
// fieldTokens returns the field and reference names a format renders via {name}
// or {a|..b} tokens (function tokens, which carry no field edges, are excluded).
// These are exactly the child nodes expand recurses into, through readField.
func fieldTokens(format string) []string {
var names []string
_ = eachToken(format, func(t ftoken) error {
if t.kind == 'b' && indexOutside(t.body, '(') < 0 {
names = append(names, splitOutside(t.body, '|')...)
}
return nil
})
return names
}
// arm is one alternative of a {a|b} token or one operand, split into the key
// `template.head` resolves and the tail of a dotted path into it. A non-empty tail
// is what makes the arm a bound draw: its head is drawn once per expansion (see
@@ -308,16 +324,6 @@ func pathArm(name, key string, segs []string) arm {
return arm{name: name, key: key, tail: segs, levels: levels, path: key + "." + strings.Join(segs, ".")}
}
// splitArms splits a token body's '|' alternatives.
func splitArms(body string, refs map[string]refBinding) []arm {
parts := splitOutside(body, '|')
arms := make([]arm, len(parts))
for i, p := range parts {
arms[i] = splitArm(p, refs)
}
return arms
}
// checkSegments rejects an unfinished path: "{a.}" and "{a..b}" each have a
// segment naming nothing. A field really named "" would otherwise make them
// resolve, so a typo would read as a path that worked.
@@ -345,9 +351,8 @@ type callFn func(s *session, emitted string, operands []string) string
// or a builtin already bound to its args. compile builds these so render never
// re-scans the format.
type op struct {
kind byte // 'l' literal run, 'f' field alternation, 'b' builtin
lit string // kind 'l'
arms []arm // kind 'f': the '|' alternatives, split into key and path once
formatToken
arms []arm // kind 'f': the '|' alternatives, split into key and path once
call callFn
// operands are the fields the builtin reads, in the order its operands func
// fixed; expand reads them before the call. nil for a builtin that reads none.
@@ -391,53 +396,41 @@ func (c *formatOps) holdName(a arm, label string) {
}
}
func (c *formatOps) function(body string, refs map[string]refBinding) {
name, args, _ := funcCall(body)
func (c *formatOps) function(tok formatToken, refs map[string]refBinding) {
var operands []arm
for _, operand := range tokenOperands(body) {
for _, operand := range tok.names {
a := splitArm(operand, refs)
c.holdName(a, fmt.Sprintf("%s operand %q", name, operand))
c.holdName(a, fmt.Sprintf("%s operand %q", tok.fn, operand))
operands = append(operands, a)
}
c.ops = append(c.ops, op{kind: 'b', call: builtins[name].prep(args), operands: operands})
c.ops = append(c.ops, op{formatToken: tok, call: builtins[tok.fn].prep(tok.args), operands: operands})
}
func (c *formatOps) field(body string, refs map[string]refBinding) {
arms := splitArms(body, refs)
for _, a := range arms {
if len(a.tail) > 0 {
c.holdName(a, "token {"+a.name+"}")
func (c *formatOps) field(tok formatToken, refs map[string]refBinding) {
arms := make([]arm, len(tok.names))
for i, name := range tok.names {
arms[i] = splitArm(name, refs)
if len(arms[i].tail) > 0 {
c.holdName(arms[i], "token {"+arms[i].name+"}")
}
}
c.ops = append(c.ops, op{kind: 'f', arms: arms})
c.ops = append(c.ops, op{formatToken: tok, arms: arms})
}
// compileOps compiles a format string. Call checkTokens first: it is what proves
// the scan and every token are valid.
func compileOps(format string, refs map[string]refBinding) formatOps {
// compileOps compiles a parsed format. Call checkTokens first: it is what proves
// every token valid.
func compileOps(toks []formatToken, refs map[string]refBinding) formatOps {
var c formatOps
var lit strings.Builder
flush := func() {
if lit.Len() > 0 {
c.grow += lit.Len()
c.ops = append(c.ops, op{kind: 'l', lit: lit.String()})
lit.Reset()
for _, tok := range toks {
switch tok.kind {
case 'l':
c.grow += len(tok.lit)
c.ops = append(c.ops, op{formatToken: tok})
case 'b':
c.function(tok, refs)
case 'f':
c.field(tok, refs)
}
}
_ = eachToken(format, func(t ftoken) error {
switch t.kind {
case 'l':
lit.WriteRune(t.r)
case 'b':
flush()
if _, _, isFunc := funcCall(t.body); isFunc {
c.function(t.body, refs)
} else {
c.field(t.body, refs)
}
}
return nil
})
flush()
return c
}
-4
View File
@@ -19,10 +19,6 @@ Self-sufficiency 5.6), every seat capped by Locality or Self-sufficiency.
This round, in order:
- Parse a format once and have the fences read `t.ops`: `eachToken` rescans the format
in `fieldTokens`, `operandTokens`, `boundReaders`, `renderEdges`, `loneRef`,
`operandReader`, `checkNoRepeatedRead` and `compileOps`, and function-versus-field is
decided by `indexOutside(body, '(')` in two of them and by `funcCall` in the rest.
- Name `pathWalk`'s modes: render, probe, check and cover are chosen today by which
callbacks are nil and whether `draws` is, and `walkChoice` returns `nil, nil` where
none is set. Make the render mode reachable only through a proved path: a walk that
+4 -4
View File
@@ -152,12 +152,12 @@ func (p *valueProof) template(t *template) proven {
v.notOperand = notOneValue(t.format, "{int()}, {float()}, {seq()}, {digits()} or {calc()}")
return v
}
body := t.format[1 : len(t.format)-1]
name, args, isFunc := funcCall(body)
o := t.ops[0]
body, name, args := o.body, o.fn, o.args
switch _, isTransform := transforms[name]; {
case !isFunc:
case o.kind == 'f':
var leaves []node
for _, a := range splitArms(body, t.refs) {
for _, a := range o.arms {
leaves = append(leaves, pathLeaves(t.head(a.key), a.tail)...)
}
return p.unite(leaves)