This commit is contained in:
Martino Ferrari
2026-04-25 22:56:09 +02:00
parent 8b548ba1c2
commit 986f6cd6d8
85 changed files with 11479 additions and 5050 deletions
@@ -0,0 +1,34 @@
// Package synthetic implements a DataSource that computes signals from
// existing upstream signals by running them through configurable DSP pipelines.
package synthetic
// SignalDef describes one synthetic signal.
type SignalDef struct {
Name string `json:"name"`
DS string `json:"ds"` // upstream data source name
Signal string `json:"signal"` // upstream signal name (or "" for constant)
Inputs []InputRef `json:"inputs"` // alternative multi-input format
Pipeline []NodeDef `json:"pipeline"` // ordered list of DSP nodes
Meta MetaOverride `json:"meta"` // optional metadata overrides
}
// InputRef names one upstream signal used as input to the pipeline.
type InputRef struct {
DS string `json:"ds"`
Signal string `json:"signal"`
}
// NodeDef is a single node in the pipeline.
type NodeDef struct {
Type string `json:"type"`
Params map[string]any `json:"params,omitempty"`
}
// MetaOverride allows the synthetic signal to override display metadata.
type MetaOverride struct {
Unit string `json:"unit,omitempty"`
Description string `json:"description,omitempty"`
DisplayLow float64 `json:"displayLow,omitempty"`
DisplayHigh float64 `json:"displayHigh,omitempty"`
Writable bool `json:"writable,omitempty"`
}
+108
View File
@@ -0,0 +1,108 @@
package synthetic
import (
"fmt"
"github.com/uopi/uopi/internal/dsp"
)
// floatParam extracts a float64 from params; returns 0 if missing or wrong type.
func floatParam(params map[string]any, key string) float64 {
if params == nil {
return 0
}
v, ok := params[key]
if !ok {
return 0
}
f, ok := v.(float64)
if !ok {
return 0
}
return f
}
// stringParam extracts a string from params; returns "" if missing or wrong type.
func stringParam(params map[string]any, key string) string {
if params == nil {
return ""
}
v, ok := params[key]
if !ok {
return ""
}
s, ok := v.(string)
if !ok {
return ""
}
return s
}
// BuildPipeline converts a []NodeDef (from JSON) into a []dsp.Node ready for
// execution. JSON numbers are float64, so all numeric params are handled as
// float64 regardless of the final type needed.
func BuildPipeline(defs []NodeDef) ([]dsp.Node, error) {
nodes := make([]dsp.Node, 0, len(defs))
for i, d := range defs {
n, err := buildNode(d)
if err != nil {
return nil, fmt.Errorf("pipeline node %d (%q): %w", i, d.Type, err)
}
nodes = append(nodes, n)
}
return nodes, nil
}
func buildNode(d NodeDef) (dsp.Node, error) {
p := d.Params
switch d.Type {
case "gain":
return &dsp.GainNode{Gain: floatParam(p, "gain")}, nil
case "offset":
return &dsp.OffsetNode{Offset: floatParam(p, "offset")}, nil
case "add":
return &dsp.AddNode{}, nil
case "subtract":
return &dsp.SubtractNode{}, nil
case "multiply":
return &dsp.MultiplyNode{}, nil
case "divide":
return &dsp.DivideNode{}, nil
case "moving_average":
return &dsp.MovingAverageNode{Window: max(1, int(floatParam(p, "window")))}, nil
case "rms":
return &dsp.RMSNode{Window: max(1, int(floatParam(p, "window")))}, nil
case "derivative":
return &dsp.DerivativeNode{}, nil
case "clamp":
return &dsp.ClampNode{
Min: floatParam(p, "min"),
Max: floatParam(p, "max"),
}, nil
case "threshold":
return &dsp.ThresholdNode{
Threshold: floatParam(p, "threshold"),
High: floatParam(p, "high"),
Low: floatParam(p, "low"),
}, nil
case "expr":
return &dsp.ExprNode{Expr: stringParam(p, "expr")}, nil
case "lua":
return &dsp.LuaNode{Script: stringParam(p, "script")}, nil
default:
return nil, fmt.Errorf("unknown node type %q", d.Type)
}
}
+451
View File
@@ -0,0 +1,451 @@
package synthetic
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"os"
"path/filepath"
"sync"
"time"
"github.com/uopi/uopi/internal/broker"
"github.com/uopi/uopi/internal/datasource"
"github.com/uopi/uopi/internal/dsp"
)
const definitionsFile = "synthetic.json"
// signalState holds everything needed to run one synthetic signal.
type signalState struct {
def SignalDef
nodes []dsp.Node
states []map[string]any // one map per node, persistent across calls
// cancel stops the goroutine driving this signal.
cancel context.CancelFunc
}
// Synthetic is a DataSource that computes values from upstream signals via
// configurable DSP pipelines.
type Synthetic struct {
brk *broker.Broker
storePath string
log *slog.Logger
mu sync.RWMutex
signals map[string]*signalState
// root context provided by Connect; used to start per-signal goroutines.
ctx context.Context
}
// New creates a Synthetic data source. storePath is the directory where
// synthetic.json is stored. brk is used to subscribe to upstream signals.
func New(storePath string, brk *broker.Broker, log *slog.Logger) *Synthetic {
return &Synthetic{
brk: brk,
storePath: storePath,
log: log,
signals: make(map[string]*signalState),
}
}
// Name implements datasource.DataSource.
func (s *Synthetic) Name() string { return "synthetic" }
// Connect loads the definitions file and starts all signal goroutines.
func (s *Synthetic) Connect(ctx context.Context) error {
s.ctx = ctx
defs, err := s.loadDefs()
if err != nil {
return fmt.Errorf("synthetic: load definitions: %w", err)
}
for _, def := range defs {
if err := s.startSignal(def); err != nil {
s.log.Warn("synthetic: failed to start signal", "name", def.Name, "err", err)
}
}
return nil
}
// ListSignals returns metadata for all defined synthetic signals.
func (s *Synthetic) ListSignals(_ context.Context) ([]datasource.Metadata, error) {
s.mu.RLock()
defer s.mu.RUnlock()
out := make([]datasource.Metadata, 0, len(s.signals))
for _, st := range s.signals {
out = append(out, defToMetadata(st.def))
}
return out, nil
}
// GetMetadata returns metadata for a single named signal.
func (s *Synthetic) GetMetadata(_ context.Context, signal string) (datasource.Metadata, error) {
s.mu.RLock()
defer s.mu.RUnlock()
st, ok := s.signals[signal]
if !ok {
return datasource.Metadata{}, datasource.ErrNotFound
}
return defToMetadata(st.def), nil
}
// Subscribe registers ch to receive computed values for the named signal.
// Values are pushed whenever an upstream signal changes.
func (s *Synthetic) Subscribe(ctx context.Context, signal string, ch chan<- datasource.Value) (datasource.CancelFunc, error) {
s.mu.RLock()
st, ok := s.signals[signal]
s.mu.RUnlock()
if !ok {
return nil, datasource.ErrNotFound
}
// Collect the upstream references for this signal.
refs := upstreamRefs(st.def)
if len(refs) == 0 {
return nil, fmt.Errorf("synthetic: signal %q has no upstream inputs", signal)
}
ctx, cancel := context.WithCancel(ctx)
go func() {
defer cancel()
// Latest value per upstream input index.
latest := make([]float64, len(refs))
ready := make([]bool, len(refs))
// Subscribe to every upstream ref via the broker.
updateCh := make(chan indexedUpdate, 32*len(refs))
unsubs := make([]func(), 0, len(refs))
for i, ref := range refs {
idx := i // capture
perCh := make(chan broker.Update, 16)
unsub, err := s.brk.Subscribe(ref, perCh)
if err != nil {
s.log.Warn("synthetic: upstream subscribe failed",
"signal", signal, "upstream", ref, "err", err)
// Continue; the slot will stay at zero.
} else {
unsubs = append(unsubs, unsub)
go func() {
for {
select {
case u, ok := <-perCh:
if !ok {
return
}
val := toFloat64(u.Value.Data)
select {
case updateCh <- indexedUpdate{idx: idx, val: val, ts: u.Value.Timestamp}:
default:
}
case <-ctx.Done():
return
}
}
}()
}
}
defer func() {
for _, unsub := range unsubs {
unsub()
}
}()
for {
select {
case <-ctx.Done():
return
case upd := <-updateCh:
latest[upd.idx] = upd.val
ready[upd.idx] = true
// Only compute once we have at least one value for every input.
allReady := true
for _, r := range ready {
if !r {
allReady = false
break
}
}
if !allReady {
continue
}
// Run the pipeline.
s.mu.RLock()
cur, stillExists := s.signals[signal]
s.mu.RUnlock()
if !stillExists {
return
}
result, err := runPipeline(cur.nodes, cur.states, latest)
if err != nil {
s.log.Warn("synthetic: pipeline error", "signal", signal, "err", err)
continue
}
v := datasource.Value{
Timestamp: upd.ts,
Data: result,
Quality: datasource.QualityGood,
}
select {
case ch <- v:
case <-ctx.Done():
return
}
}
}
}()
return datasource.CancelFunc(cancel), nil
}
// Write is not supported for synthetic signals.
func (s *Synthetic) Write(_ context.Context, _ string, _ any) error {
return datasource.ErrNotWritable
}
// History is not supported for synthetic signals.
func (s *Synthetic) History(_ context.Context, _ string, _, _ time.Time, _ int) ([]datasource.Value, error) {
return nil, datasource.ErrHistoryUnavailable
}
// AddSignal adds a new synthetic signal definition at runtime and persists it.
func (s *Synthetic) AddSignal(def SignalDef) error {
if def.Name == "" {
return errors.New("signal name must not be empty")
}
nodes, err := BuildPipeline(def.Pipeline)
if err != nil {
return fmt.Errorf("build pipeline: %w", err)
}
s.mu.Lock()
if _, exists := s.signals[def.Name]; exists {
s.mu.Unlock()
return fmt.Errorf("signal %q already exists", def.Name)
}
states := make([]map[string]any, len(nodes))
for i := range states {
states[i] = make(map[string]any)
}
st := &signalState{
def: def,
nodes: nodes,
states: states,
}
s.signals[def.Name] = st
s.mu.Unlock()
if err := s.saveDefs(); err != nil {
// Roll back the in-memory addition.
s.mu.Lock()
delete(s.signals, def.Name)
s.mu.Unlock()
return fmt.Errorf("persist definitions: %w", err)
}
return nil
}
// RemoveSignal removes a synthetic signal definition at runtime and persists.
func (s *Synthetic) RemoveSignal(name string) error {
s.mu.Lock()
st, ok := s.signals[name]
if !ok {
s.mu.Unlock()
return datasource.ErrNotFound
}
// Stop the goroutine if it is running.
if st.cancel != nil {
st.cancel()
}
delete(s.signals, name)
s.mu.Unlock()
if err := s.saveDefs(); err != nil {
return fmt.Errorf("persist definitions: %w", err)
}
return nil
}
// GetDefs returns a copy of all current signal definitions (for the REST API).
func (s *Synthetic) GetDefs() []SignalDef {
s.mu.RLock()
defer s.mu.RUnlock()
out := make([]SignalDef, 0, len(s.signals))
for _, st := range s.signals {
out = append(out, st.def)
}
return out
}
// ── internal helpers ──────────────────────────────────────────────────────────
func (s *Synthetic) defsFilePath() string {
return filepath.Join(s.storePath, definitionsFile)
}
func (s *Synthetic) loadDefs() ([]SignalDef, error) {
path := s.defsFilePath()
data, err := os.ReadFile(path)
if errors.Is(err, os.ErrNotExist) {
// Create an empty definitions file.
if werr := os.WriteFile(path, []byte("[]\n"), 0o644); werr != nil {
s.log.Warn("synthetic: could not create empty definitions file", "path", path, "err", werr)
}
return nil, nil
}
if err != nil {
return nil, err
}
var defs []SignalDef
if err := json.Unmarshal(data, &defs); err != nil {
return nil, fmt.Errorf("parse %s: %w", path, err)
}
return defs, nil
}
func (s *Synthetic) saveDefs() error {
s.mu.RLock()
defs := make([]SignalDef, 0, len(s.signals))
for _, st := range s.signals {
defs = append(defs, st.def)
}
s.mu.RUnlock()
data, err := json.MarshalIndent(defs, "", " ")
if err != nil {
return err
}
return os.WriteFile(s.defsFilePath(), data, 0o644)
}
// startSignal builds the pipeline for def and registers the signalState.
// The actual goroutines are started lazily by Subscribe.
func (s *Synthetic) startSignal(def SignalDef) error {
nodes, err := BuildPipeline(def.Pipeline)
if err != nil {
return fmt.Errorf("build pipeline for %q: %w", def.Name, err)
}
states := make([]map[string]any, len(nodes))
for i := range states {
states[i] = make(map[string]any)
}
s.mu.Lock()
s.signals[def.Name] = &signalState{
def: def,
nodes: nodes,
states: states,
}
s.mu.Unlock()
s.log.Info("synthetic: signal registered", "name", def.Name)
return nil
}
// runPipeline executes all nodes in sequence. The output of node N becomes
// input[0] of node N+1. For the first node, inputs is the full upstream slice.
func runPipeline(nodes []dsp.Node, states []map[string]any, inputs []float64) (float64, error) {
if len(nodes) == 0 {
if len(inputs) == 0 {
return 0, nil
}
return inputs[0], nil
}
// First node receives all upstream inputs.
cur := inputs
var result float64
var err error
for i, node := range nodes {
result, err = node.Process(cur, states[i])
if err != nil {
return 0, fmt.Errorf("node %d (%s): %w", i, node.Type(), err)
}
// Subsequent nodes receive only the single output of the previous node.
cur = []float64{result}
}
return result, nil
}
// upstreamRefs returns the broker.SignalRef list for a SignalDef.
// If Inputs is set, those take precedence; otherwise DS+Signal is used.
func upstreamRefs(def SignalDef) []broker.SignalRef {
if len(def.Inputs) > 0 {
refs := make([]broker.SignalRef, len(def.Inputs))
for i, inp := range def.Inputs {
refs[i] = broker.SignalRef{DS: inp.DS, Name: inp.Signal}
}
return refs
}
if def.DS != "" && def.Signal != "" {
return []broker.SignalRef{{DS: def.DS, Name: def.Signal}}
}
return nil
}
// defToMetadata converts a SignalDef into a datasource.Metadata.
func defToMetadata(def SignalDef) datasource.Metadata {
return datasource.Metadata{
Name: def.Name,
Type: datasource.TypeFloat64,
Unit: def.Meta.Unit,
Description: def.Meta.Description,
DisplayLow: def.Meta.DisplayLow,
DisplayHigh: def.Meta.DisplayHigh,
Writable: def.Meta.Writable,
}
}
// toFloat64 coerces any numeric value from a datasource.Value.Data to float64.
func toFloat64(v any) float64 {
switch val := v.(type) {
case float64:
return val
case float32:
return float64(val)
case int64:
return float64(val)
case int32:
return float64(val)
case int:
return float64(val)
case bool:
if val {
return 1
}
return 0
default:
return 0
}
}
// indexedUpdate carries a value from one upstream goroutine to the pipeline runner.
type indexedUpdate struct {
idx int
val float64
ts time.Time
}
@@ -0,0 +1,259 @@
package synthetic
import (
"context"
"log/slog"
"os"
"path/filepath"
"testing"
"time"
"github.com/uopi/uopi/internal/broker"
"github.com/uopi/uopi/internal/datasource"
)
func newTestSynthetic(t *testing.T) (*Synthetic, *broker.Broker, context.CancelFunc) {
t.Helper()
dir := t.TempDir()
log := slog.New(slog.NewTextHandler(os.Stderr, nil))
ctx, cancel := context.WithCancel(context.Background())
brk := broker.New(ctx, log)
syn := New(dir, brk, log)
return syn, brk, cancel
}
// TestNameAndConnect verifies basic construction and Connect.
func TestNameAndConnect(t *testing.T) {
syn, _, cancel := newTestSynthetic(t)
defer cancel()
if syn.Name() != "synthetic" {
t.Errorf("Name: want %q, got %q", "synthetic", syn.Name())
}
ctx := context.Background()
if err := syn.Connect(ctx); err != nil {
t.Fatalf("Connect: %v", err)
}
}
// TestConnectCreatesEmptyFile verifies that Connect creates an empty
// synthetic.json when none exists.
func TestConnectCreatesEmptyFile(t *testing.T) {
syn, _, cancel := newTestSynthetic(t)
defer cancel()
if err := syn.Connect(context.Background()); err != nil {
t.Fatal(err)
}
path := filepath.Join(syn.storePath, "synthetic.json")
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("expected file to be created: %v", err)
}
if string(data) == "" {
t.Error("file should not be empty")
}
}
// TestAddAndRemoveSignal exercises CRUD at runtime.
func TestAddAndRemoveSignal(t *testing.T) {
syn, _, cancel := newTestSynthetic(t)
defer cancel()
if err := syn.Connect(context.Background()); err != nil {
t.Fatal(err)
}
def := SignalDef{
Name: "test_gain",
DS: "stub",
Signal: "sine_1hz",
Pipeline: []NodeDef{
{Type: "gain", Params: map[string]any{"gain": 2.0}},
},
Meta: MetaOverride{Unit: "V"},
}
if err := syn.AddSignal(def); err != nil {
t.Fatalf("AddSignal: %v", err)
}
// Duplicate should fail.
if err := syn.AddSignal(def); err == nil {
t.Error("expected error for duplicate signal")
}
// Should be listed.
metas, err := syn.ListSignals(context.Background())
if err != nil {
t.Fatal(err)
}
if len(metas) != 1 || metas[0].Name != "test_gain" {
t.Errorf("unexpected metas: %v", metas)
}
// GetMetadata should work.
meta, err := syn.GetMetadata(context.Background(), "test_gain")
if err != nil {
t.Fatalf("GetMetadata: %v", err)
}
if meta.Unit != "V" {
t.Errorf("unit: want V, got %q", meta.Unit)
}
// Remove it.
if err := syn.RemoveSignal("test_gain"); err != nil {
t.Fatalf("RemoveSignal: %v", err)
}
// Should be gone.
if _, err := syn.GetMetadata(context.Background(), "test_gain"); err != datasource.ErrNotFound {
t.Errorf("expected ErrNotFound after remove, got %v", err)
}
// Remove non-existing should return ErrNotFound.
if err := syn.RemoveSignal("nonexistent"); err != datasource.ErrNotFound {
t.Errorf("expected ErrNotFound, got %v", err)
}
}
// TestAddSignalEmptyName validates that empty name is rejected.
func TestAddSignalEmptyName(t *testing.T) {
syn, _, cancel := newTestSynthetic(t)
defer cancel()
if err := syn.Connect(context.Background()); err != nil {
t.Fatal(err)
}
if err := syn.AddSignal(SignalDef{}); err == nil {
t.Error("expected error for empty signal name")
}
}
// TestGetMetadataNotFound verifies ErrNotFound for unknown signal.
func TestGetMetadataNotFound(t *testing.T) {
syn, _, cancel := newTestSynthetic(t)
defer cancel()
if err := syn.Connect(context.Background()); err != nil {
t.Fatal(err)
}
_, err := syn.GetMetadata(context.Background(), "no_such_signal")
if err != datasource.ErrNotFound {
t.Errorf("expected ErrNotFound, got %v", err)
}
}
// TestWriteNotSupported ensures Write returns ErrNotWritable.
func TestWriteNotSupported(t *testing.T) {
syn, _, cancel := newTestSynthetic(t)
defer cancel()
err := syn.Write(context.Background(), "any", 1.0)
if err != datasource.ErrNotWritable {
t.Errorf("expected ErrNotWritable, got %v", err)
}
}
// TestHistoryNotSupported ensures History returns ErrHistoryUnavailable.
func TestHistoryNotSupported(t *testing.T) {
syn, _, cancel := newTestSynthetic(t)
defer cancel()
_, err := syn.History(context.Background(), "any", time.Now(), time.Now(), 10)
if err != datasource.ErrHistoryUnavailable {
t.Errorf("expected ErrHistoryUnavailable, got %v", err)
}
}
// TestPersistence verifies that definitions survive a restart (New+Connect).
func TestPersistence(t *testing.T) {
dir := t.TempDir()
log := slog.New(slog.NewTextHandler(os.Stderr, nil))
ctx, cancel := context.WithCancel(context.Background())
brk := broker.New(ctx, log)
syn := New(dir, brk, log)
if err := syn.Connect(ctx); err != nil {
t.Fatal(err)
}
def := SignalDef{
Name: "persisted",
DS: "stub",
Signal: "sine_1hz",
Pipeline: []NodeDef{
{Type: "offset", Params: map[string]any{"offset": 5.0}},
},
}
if err := syn.AddSignal(def); err != nil {
t.Fatal(err)
}
cancel()
// Reload.
ctx2 := t.Context()
brk2 := broker.New(ctx2, log)
syn2 := New(dir, brk2, log)
if err := syn2.Connect(ctx2); err != nil {
t.Fatal(err)
}
meta, err := syn2.GetMetadata(ctx2, "persisted")
if err != nil {
t.Fatalf("after reload, GetMetadata: %v", err)
}
if meta.Name != "persisted" {
t.Errorf("expected name %q, got %q", "persisted", meta.Name)
}
}
// TestSubscribeUnknownSignal verifies that subscribing to unknown signal returns error.
func TestSubscribeUnknownSignal(t *testing.T) {
syn, _, cancel := newTestSynthetic(t)
defer cancel()
if err := syn.Connect(context.Background()); err != nil {
t.Fatal(err)
}
ch := make(chan datasource.Value, 1)
_, err := syn.Subscribe(context.Background(), "nonexistent", ch)
if err != datasource.ErrNotFound {
t.Errorf("expected ErrNotFound, got %v", err)
}
}
// TestGetDefs verifies GetDefs returns current definitions.
func TestGetDefs(t *testing.T) {
syn, _, cancel := newTestSynthetic(t)
defer cancel()
if err := syn.Connect(context.Background()); err != nil {
t.Fatal(err)
}
defs := syn.GetDefs()
if len(defs) != 0 {
t.Errorf("expected empty defs, got %d", len(defs))
}
def := SignalDef{
Name: "test",
DS: "stub",
Signal: "noise",
}
if err := syn.AddSignal(def); err != nil {
t.Fatal(err)
}
defs = syn.GetDefs()
if len(defs) != 1 || defs[0].Name != "test" {
t.Errorf("unexpected defs: %v", defs)
}
}