Implemented confi snapshots

This commit is contained in:
Martino Ferrari
2026-06-21 23:50:03 +02:00
parent 04d31a15c4
commit 113e5a0fe8
51 changed files with 4921 additions and 241 deletions
+199
View File
@@ -0,0 +1,199 @@
package confmgr
import (
"fmt"
"reflect"
"strings"
"cuelang.org/go/cue"
"cuelang.org/go/cue/cuecontext"
cueerrors "cuelang.org/go/cue/errors"
)
// ConfigRule is a CUE-based validation/transformation rule bound to a config
// set. Its Source is CUE describing constraints and/or derivations over the
// instance's parameter values. Regular CUE fields whose key matches a set
// parameter are unified with the instance value; constraints that fail produce
// violations, and concrete fields that differ from the instance value are
// reported (and persisted) as transformations. Hidden fields (_x) and
// definitions (#X) are available for helpers and are excluded from both
// concreteness checks and transformation output. Rules are versioned git-style,
// exactly like sets and instances.
type ConfigRule struct {
ID string `json:"id"`
Name string `json:"name"`
SetID string `json:"setId"`
Description string `json:"description,omitempty"`
Version int `json:"version"`
Tag string `json:"tag,omitempty"`
Owner string `json:"owner,omitempty"`
Source string `json:"source"`
}
// RuleViolation is a single constraint failure from evaluating a rule.
type RuleViolation struct {
Rule string `json:"rule,omitempty"` // rule ID that produced it (aggregate runs)
Path string `json:"path,omitempty"` // dotted parameter path, when known
Message string `json:"message"`
}
// RuleResult is the outcome of evaluating one or more rules against a set of
// instance values. OK is true only when there is no compile error and no
// violation. Transformed holds the parameter values the rule(s) derived or
// overrode (keys are parameter keys; values are the new concrete values).
type RuleResult struct {
OK bool `json:"ok"`
Violations []RuleViolation `json:"violations,omitempty"`
Transformed map[string]any `json:"transformed,omitempty"`
CompileError string `json:"compileError,omitempty"`
}
// RuleError wraps a failing RuleResult so it can flow through the store's
// error-returning API while preserving the structured violations.
type RuleError struct {
Result RuleResult
}
func (e *RuleError) Error() string {
if e.Result.CompileError != "" {
return "rule compile error: " + e.Result.CompileError
}
msgs := make([]string, 0, len(e.Result.Violations))
for _, v := range e.Result.Violations {
if v.Path != "" {
msgs = append(msgs, v.Path+": "+v.Message)
} else {
msgs = append(msgs, v.Message)
}
}
if len(msgs) == 0 {
return "rule validation failed"
}
return "rule validation failed: " + strings.Join(msgs, "; ")
}
// Validate compiles the rule's CUE source and checks basic metadata. It does
// not require concrete values — only that the source is syntactically and
// structurally well-formed CUE.
func (r ConfigRule) Validate() error {
if strings.TrimSpace(r.Name) == "" {
return fmt.Errorf("rule name must not be empty")
}
if strings.TrimSpace(r.SetID) == "" {
return fmt.Errorf("rule must reference a config set")
}
ctx := cuecontext.New()
v := ctx.CompileString(r.Source)
if err := v.Err(); err != nil {
return fmt.Errorf("invalid CUE: %s", firstError(err))
}
return nil
}
// EvaluateRule unifies a single CUE source with the given instance values and
// reports violations plus any transformed values. A compile error yields a
// non-OK result with CompileError populated (and no violations), so callers can
// distinguish "the rule is broken" from "the values are invalid".
func EvaluateRule(source string, values map[string]any) RuleResult {
ctx := cuecontext.New()
schema := ctx.CompileString(source)
if err := schema.Err(); err != nil {
return RuleResult{OK: false, CompileError: firstError(err)}
}
data := ctx.Encode(values)
if err := data.Err(); err != nil {
return RuleResult{OK: false, CompileError: "cannot encode values: " + firstError(err)}
}
unified := schema.Unify(data)
res := RuleResult{OK: true}
if err := unified.Validate(cue.Concrete(true), cue.All()); err != nil {
res.OK = false
for _, e := range cueerrors.Errors(err) {
res.Violations = append(res.Violations, RuleViolation{
Path: strings.Join(e.Path(), "."),
Message: cleanMessage(e),
})
}
if len(res.Violations) == 0 {
res.Violations = append(res.Violations, RuleViolation{Message: firstError(err)})
}
return res
}
// Extract transformations: regular concrete fields whose value differs from
// the supplied input. Definitions and hidden fields are skipped by Fields().
iter, err := unified.Fields()
if err != nil {
return res
}
for iter.Next() {
key := iter.Selector().Unquoted()
if key == "" {
key = strings.Trim(iter.Selector().String(), `"`)
}
var v any
if err := iter.Value().Decode(&v); err != nil {
continue
}
if !reflect.DeepEqual(v, values[key]) {
if res.Transformed == nil {
res.Transformed = map[string]any{}
}
res.Transformed[key] = v
}
}
return res
}
// evaluateRules runs several rules against the same values and aggregates the
// outcome. Violations are tagged with the rule ID. Transformations are applied
// cumulatively in order; a later rule sees earlier rules' outputs.
func evaluateRules(rules []ConfigRule, values map[string]any) RuleResult {
agg := RuleResult{OK: true}
cur := make(map[string]any, len(values))
for k, v := range values {
cur[k] = v
}
for _, r := range rules {
one := EvaluateRule(r.Source, cur)
if one.CompileError != "" {
agg.OK = false
agg.Violations = append(agg.Violations, RuleViolation{Rule: r.ID, Message: "compile error: " + one.CompileError})
continue
}
if !one.OK {
agg.OK = false
for _, v := range one.Violations {
v.Rule = r.ID
agg.Violations = append(agg.Violations, v)
}
continue
}
for k, v := range one.Transformed {
if agg.Transformed == nil {
agg.Transformed = map[string]any{}
}
agg.Transformed[k] = v
cur[k] = v
}
}
return agg
}
// firstError renders the first CUE error as a single line.
func firstError(err error) string {
errs := cueerrors.Errors(err)
if len(errs) == 0 {
return err.Error()
}
return cleanMessage(errs[0])
}
// cleanMessage formats a CUE error to a compact single-line string without the
// noisy file:line position prefix that cuecontext synthesises for anonymous
// sources.
func cleanMessage(e cueerrors.Error) string {
format, args := e.Msg()
return fmt.Sprintf(format, args...)
}
+80
View File
@@ -0,0 +1,80 @@
package confmgr
import "testing"
func TestEvaluateRule_Valid(t *testing.T) {
src := `
current_limit: >=0 & <=max
max: 100
`
res := EvaluateRule(src, map[string]any{"current_limit": 50.0})
if !res.OK {
t.Fatalf("expected OK, got violations: %+v compile=%q", res.Violations, res.CompileError)
}
if res.CompileError != "" {
t.Fatalf("unexpected compile error: %s", res.CompileError)
}
}
func TestEvaluateRule_Violation(t *testing.T) {
src := `current_limit: >=0 & <=100`
res := EvaluateRule(src, map[string]any{"current_limit": 150.0})
if res.OK {
t.Fatalf("expected violation, got OK")
}
if len(res.Violations) == 0 {
t.Fatalf("expected at least one violation")
}
}
func TestEvaluateRule_Transform(t *testing.T) {
// power is derived from the supplied current & voltage.
src := `
current: number
voltage: number
power: current * voltage
`
res := EvaluateRule(src, map[string]any{"current": 2.0, "voltage": 3.0})
if !res.OK {
t.Fatalf("expected OK, got %+v", res.Violations)
}
got, ok := res.Transformed["power"]
if !ok {
t.Fatalf("expected transformed power, got %+v", res.Transformed)
}
if f, _ := toFloat(got); f != 6 {
t.Fatalf("expected power=6, got %v", got)
}
}
func TestEvaluateRule_DefaultFillsMissing(t *testing.T) {
src := `mode: *"auto" | "manual"`
res := EvaluateRule(src, map[string]any{})
if !res.OK {
t.Fatalf("expected OK, got %+v", res.Violations)
}
if res.Transformed["mode"] != "auto" {
t.Fatalf("expected mode default auto, got %v", res.Transformed["mode"])
}
}
func TestEvaluateRule_CompileError(t *testing.T) {
res := EvaluateRule(`this is : : not cue`, map[string]any{})
if res.OK || res.CompileError == "" {
t.Fatalf("expected compile error, got %+v", res)
}
}
func TestConfigRule_Validate(t *testing.T) {
r := ConfigRule{Name: "r", SetID: "s", Source: `x: >=0`}
if err := r.Validate(); err != nil {
t.Fatalf("expected valid rule, got %v", err)
}
bad := ConfigRule{Name: "r", SetID: "s", Source: `x: : :`}
if err := bad.Validate(); err == nil {
t.Fatalf("expected compile error for bad source")
}
if err := (ConfigRule{Source: `x: 1`}).Validate(); err == nil {
t.Fatalf("expected error for missing name/setID")
}
}
+129
View File
@@ -0,0 +1,129 @@
package confmgr
import (
"fmt"
"math"
"slices"
"strconv"
)
// ReadFunc reads the current raw value of a target signal. The API/engine layer
// supplies a closure backed by the broker (ReadNow) so confmgr stays decoupled
// from the transport. The returned value mirrors datasource.Value.Data
// (float64 | []float64 | string | int64 | bool; enums arrive as an int64 index).
type ReadFunc func(ds, signal string) (any, error)
// SnapshotEntry records the outcome of reading one parameter's target signal.
type SnapshotEntry struct {
Key string `json:"key"`
DS string `json:"ds"`
Signal string `json:"signal"`
Value any `json:"value,omitempty"`
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
}
// SnapshotResult summarises a snapshot run. Values holds the captured,
// type-coerced values ready to populate a new ConfigInstance.
type SnapshotResult struct {
SetID string `json:"setId"`
Entries []SnapshotEntry `json:"entries"`
Captured int `json:"captured"`
Failed int `json:"failed"`
Values map[string]any `json:"-"`
}
// Snapshot reads the current value of every parameter's target signal via read
// and builds a value map for a new instance. Read failures and values that
// cannot be coerced to the parameter's type are recorded per-entry rather than
// aborting the whole snapshot, so a partial capture is reported faithfully.
func Snapshot(set ConfigSet, read ReadFunc) SnapshotResult {
res := SnapshotResult{
SetID: set.ID,
Entries: make([]SnapshotEntry, 0, len(set.Parameters)),
Values: make(map[string]any, len(set.Parameters)),
}
for _, p := range set.Parameters {
e := SnapshotEntry{Key: p.Key, DS: p.DS, Signal: p.Signal}
raw, err := read(p.DS, p.Signal)
if err != nil {
e.Error = err.Error()
res.Failed++
res.Entries = append(res.Entries, e)
continue
}
v, err := p.coerceSnapshot(raw)
if err != nil {
e.Error = err.Error()
res.Failed++
res.Entries = append(res.Entries, e)
continue
}
e.Value = v
e.OK = true
res.Values[p.Key] = v
res.Captured++
res.Entries = append(res.Entries, e)
}
return res
}
// coerceSnapshot converts a raw datasource value into the canonical Go value the
// parameter's type expects, so the result passes checkValue and serialises
// cleanly. Enum signals arrive as an int64 index and are mapped to their string.
func (p Parameter) coerceSnapshot(raw any) (any, error) {
switch p.Type {
case TypeFloat:
return toFloat(raw)
case TypeInt:
f, err := toFloat(raw)
if err != nil {
return nil, err
}
return int64(math.Round(f)), nil
case TypeBool:
switch b := raw.(type) {
case bool:
return b, nil
case string:
pb, err := strconv.ParseBool(b)
if err != nil {
return nil, fmt.Errorf("value %q is not a bool", b)
}
return pb, nil
default:
f, err := toFloat(raw)
if err != nil {
return nil, fmt.Errorf("value %v is not a bool", raw)
}
return f != 0, nil
}
case TypeString:
if s, ok := raw.(string); ok {
return s, nil
}
return fmt.Sprintf("%v", raw), nil
case TypeEnum:
// Enum signals deliver an int64 index into the metadata strings; map it
// to this parameter's enum value. A string already in range is kept.
if s, ok := raw.(string); ok {
if slices.Contains(p.EnumValues, s) {
return s, nil
}
return nil, fmt.Errorf("value %q is not an allowed enum value", s)
}
f, err := toFloat(raw)
if err != nil {
return nil, fmt.Errorf("enum value %v is not an index", raw)
}
idx := int(math.Round(f))
if idx < 0 || idx >= len(p.EnumValues) {
return nil, fmt.Errorf("enum index %d out of range", idx)
}
return p.EnumValues[idx], nil
case TypeFloatArray:
return toFloatArray(raw)
default:
return raw, nil
}
}
+83
View File
@@ -0,0 +1,83 @@
package confmgr
import (
"errors"
"testing"
)
func TestSnapshotCapturesAndCoerces(t *testing.T) {
set := ConfigSet{
ID: "set1",
Name: "s",
Parameters: []Parameter{
{Key: "v", DS: "epics", Signal: "PSU:V", Type: TypeFloat},
{Key: "n", DS: "epics", Signal: "PSU:N", Type: TypeInt},
{Key: "en", DS: "epics", Signal: "PSU:EN", Type: TypeBool},
{Key: "mode", DS: "epics", Signal: "PSU:MODE", Type: TypeEnum, EnumValues: []string{"off", "low", "high"}},
{Key: "wf", DS: "epics", Signal: "PSU:WF", Type: TypeFloatArray},
{Key: "lbl", DS: "epics", Signal: "PSU:LBL", Type: TypeString},
},
}
live := map[string]any{
"PSU:V": 24.5,
"PSU:N": 3.0, // float reading → coerced to int64
"PSU:EN": int64(1), // numeric → bool true
"PSU:MODE": int64(2), // enum index → "high"
"PSU:WF": []float64{1, 2, 3},
"PSU:LBL": "ready",
}
res := Snapshot(set, func(_, signal string) (any, error) {
v, ok := live[signal]
if !ok {
return nil, errors.New("not read")
}
return v, nil
})
if res.Captured != 6 || res.Failed != 0 {
t.Fatalf("summary: captured=%d failed=%d", res.Captured, res.Failed)
}
if res.Values["v"] != 24.5 {
t.Errorf("v = %v, want 24.5", res.Values["v"])
}
if res.Values["n"] != int64(3) {
t.Errorf("n = %v (%T), want int64(3)", res.Values["n"], res.Values["n"])
}
if res.Values["en"] != true {
t.Errorf("en = %v, want true", res.Values["en"])
}
if res.Values["mode"] != "high" {
t.Errorf("mode = %v, want high", res.Values["mode"])
}
if res.Values["lbl"] != "ready" {
t.Errorf("lbl = %v, want ready", res.Values["lbl"])
}
// The captured values must validate against the set.
inst := ConfigInstance{SetID: set.ID, Values: res.Values}
if err := inst.ValidateAgainst(set); err != nil {
t.Errorf("snapshot values fail validation: %v", err)
}
}
func TestSnapshotReadFailureRecorded(t *testing.T) {
set := ConfigSet{
ID: "set1",
Name: "s",
Parameters: []Parameter{
{Key: "a", DS: "epics", Signal: "A", Type: TypeFloat},
{Key: "b", DS: "epics", Signal: "B", Type: TypeFloat},
},
}
res := Snapshot(set, func(_, signal string) (any, error) {
if signal == "B" {
return nil, errors.New("offline")
}
return 1.0, nil
})
if res.Captured != 1 || res.Failed != 1 {
t.Fatalf("summary: captured=%d failed=%d", res.Captured, res.Failed)
}
if _, ok := res.Values["b"]; ok {
t.Errorf("failed parameter must not appear in values")
}
}
+138 -4
View File
@@ -22,13 +22,18 @@ type Kind int
const (
KindSet Kind = iota
KindInstance
KindRule
)
func (k Kind) sub() string {
if k == KindInstance {
switch k {
case KindInstance:
return "instances"
case KindRule:
return "rules"
default:
return "sets"
}
return "sets"
}
// Meta is the lightweight listing representation.
@@ -36,6 +41,10 @@ type Meta struct {
ID string `json:"id"`
Name string `json:"name"`
Version int `json:"version"`
// SetID is only populated for instances (the schema they belong to); it is
// empty for sets. Lets clients group/filter instances by set without a GET
// per instance.
SetID string `json:"setId,omitempty"`
}
// VersionMeta describes a single persisted revision.
@@ -59,7 +68,7 @@ type Store struct {
// New opens (and creates, if needed) the configs storage directories.
func New(storageDir string) (*Store, error) {
s := &Store{rootDir: storageDir, dirs: map[Kind]string{}}
for _, k := range []Kind{KindSet, KindInstance} {
for _, k := range []Kind{KindSet, KindInstance, KindRule} {
dir := filepath.Join(storageDir, "configs", k.sub())
if err := os.MkdirAll(dir, 0o755); err != nil {
return nil, fmt.Errorf("create %s dir: %w", k.sub(), err)
@@ -75,6 +84,7 @@ type header struct {
Name string `json:"name"`
Version int `json:"version"`
Tag string `json:"tag"`
SetID string `json:"setId"`
}
func validateID(id string) error {
@@ -139,7 +149,7 @@ func (s *Store) List(k Kind) ([]Meta, error) {
if err != nil {
continue
}
out = append(out, Meta{ID: id, Name: h.Name, Version: h.Version})
out = append(out, Meta{ID: id, Name: h.Name, Version: h.Version, SetID: h.SetID})
}
return out, nil
}
@@ -462,6 +472,9 @@ func (s *Store) CreateInstance(inst ConfigInstance, tag string) (ConfigInstance,
if err := inst.ValidateAgainst(set); err != nil {
return ConfigInstance{}, err
}
if err := s.applyRules(&inst, set); err != nil {
return ConfigInstance{}, err
}
obj, err := toMap(inst)
if err != nil {
return ConfigInstance{}, err
@@ -483,6 +496,9 @@ func (s *Store) UpdateInstance(id string, inst ConfigInstance, tag string) (Conf
if err := inst.ValidateAgainst(set); err != nil {
return ConfigInstance{}, err
}
if err := s.applyRules(&inst, set); err != nil {
return ConfigInstance{}, err
}
obj, err := toMap(inst)
if err != nil {
return ConfigInstance{}, err
@@ -500,6 +516,124 @@ func (s *Store) SetForInstance(inst ConfigInstance) (ConfigSet, error) {
return s.setForInstance(inst)
}
// ── rules ────────────────────────────────────────────────────────────────────
// GetRule returns the current revision of a CUE validation/transformation rule.
func (s *Store) GetRule(id string) (ConfigRule, error) {
data, err := s.get(KindRule, id)
if err != nil {
return ConfigRule{}, err
}
var rule ConfigRule
return rule, json.Unmarshal(data, &rule)
}
// GetRuleVersion returns a specific revision of a rule.
func (s *Store) GetRuleVersion(id string, version int) (ConfigRule, error) {
data, err := s.getVersion(KindRule, id, version)
if err != nil {
return ConfigRule{}, err
}
var rule ConfigRule
return rule, json.Unmarshal(data, &rule)
}
// CreateRule validates the rule's CUE source and stores it.
func (s *Store) CreateRule(rule ConfigRule, tag string) (ConfigRule, error) {
if err := rule.Validate(); err != nil {
return ConfigRule{}, err
}
obj, err := toMap(rule)
if err != nil {
return ConfigRule{}, err
}
_, data, err := s.create(KindRule, obj, tag)
if err != nil {
return ConfigRule{}, err
}
var out ConfigRule
return out, json.Unmarshal(data, &out)
}
// UpdateRule validates and stores a new revision of an existing rule.
func (s *Store) UpdateRule(id string, rule ConfigRule, tag string) (ConfigRule, error) {
if err := rule.Validate(); err != nil {
return ConfigRule{}, err
}
obj, err := toMap(rule)
if err != nil {
return ConfigRule{}, err
}
data, err := s.update(KindRule, id, obj, tag)
if err != nil {
return ConfigRule{}, err
}
var out ConfigRule
return out, json.Unmarshal(data, &out)
}
// rulesForSet loads every current-revision rule bound to the given set.
func (s *Store) rulesForSet(setID string) ([]ConfigRule, error) {
metas, err := s.List(KindRule)
if err != nil {
return nil, err
}
var out []ConfigRule
for _, m := range metas {
if m.SetID != setID {
continue
}
rule, err := s.GetRule(m.ID)
if err != nil {
continue
}
out = append(out, rule)
}
return out, nil
}
// applyRules runs every rule bound to inst's set against its values. On
// success it merges rule-derived values back into inst (parameter keys only) so
// the stored instance reflects the canonical, transformed configuration. A
// violation is returned as *RuleError so the caller can surface the structured
// details.
func (s *Store) applyRules(inst *ConfigInstance, set ConfigSet) error {
rules, err := s.rulesForSet(inst.SetID)
if err != nil {
return err
}
if len(rules) == 0 {
return nil
}
res := evaluateRules(rules, inst.Values)
if !res.OK {
return &RuleError{Result: res}
}
for k, v := range res.Transformed {
if _, ok := set.param(k); ok {
if inst.Values == nil {
inst.Values = map[string]any{}
}
inst.Values[k] = v
}
}
return nil
}
// ValidateInstanceRules evaluates the stored rules for an instance without
// persisting anything. It is the read-only counterpart used by the validate
// endpoint.
func (s *Store) ValidateInstanceRules(inst ConfigInstance) (RuleResult, error) {
rules, err := s.rulesForSet(inst.SetID)
if err != nil {
return RuleResult{}, err
}
if len(rules) == 0 {
return RuleResult{OK: true}, nil
}
return evaluateRules(rules, inst.Values), nil
}
// ── JSON helpers ────────────────────────────────────────────────────────────
func name(obj map[string]any) string {
+70
View File
@@ -41,6 +41,76 @@ func TestCreateAndGetSet(t *testing.T) {
}
}
func TestRuleBlocksInvalidInstance(t *testing.T) {
s, _ := New(t.TempDir())
set, _ := s.CreateSet(sampleSet(), "")
if _, err := s.CreateRule(ConfigRule{
Name: "voltage cap",
SetID: set.ID,
Source: "voltage: <=24",
}, ""); err != nil {
t.Fatal(err)
}
// In-range value passes.
if _, err := s.CreateInstance(ConfigInstance{
Name: "ok", SetID: set.ID, Values: map[string]any{"voltage": 20.0},
}, ""); err != nil {
t.Fatalf("expected in-range instance to save, got %v", err)
}
// Out-of-range value is rejected by the rule.
_, err := s.CreateInstance(ConfigInstance{
Name: "bad", SetID: set.ID, Values: map[string]any{"voltage": 40.0},
}, "")
if err == nil {
t.Fatalf("expected rule violation to block save")
}
var re *RuleError
if !asRuleError(err, &re) {
t.Fatalf("expected *RuleError, got %T: %v", err, err)
}
}
func TestRuleTransformPersists(t *testing.T) {
s, _ := New(t.TempDir())
set := sampleSet()
max := 48.0
set.Parameters = append(set.Parameters, Parameter{Key: "vmax", DS: "epics", Signal: "PSU:VMAX", Type: TypeFloat, Min: &max})
created, _ := s.CreateSet(set, "")
// Rule derives vmax = voltage * 2 (within range for voltage=20 -> 40).
if _, err := s.CreateRule(ConfigRule{
Name: "vmax derive", SetID: created.ID, Source: "voltage: number\nvmax: voltage * 2",
}, ""); err != nil {
t.Fatal(err)
}
inst, err := s.CreateInstance(ConfigInstance{
Name: "x", SetID: created.ID, Values: map[string]any{"voltage": 20.0},
}, "")
if err != nil {
t.Fatal(err)
}
if f, _ := toFloat(inst.Values["vmax"]); f != 40 {
t.Fatalf("expected derived vmax=40 to persist, got %v", inst.Values["vmax"])
}
}
func asRuleError(err error, target **RuleError) bool {
for err != nil {
if re, ok := err.(*RuleError); ok {
*target = re
return true
}
type unwrapper interface{ Unwrap() error }
u, ok := err.(unwrapper)
if !ok {
return false
}
err = u.Unwrap()
}
return false
}
func TestSetVersioning(t *testing.T) {
s, _ := New(t.TempDir())
created, _ := s.CreateSet(sampleSet(), "")