Improving all side of app
This commit is contained in:
Martino Ferrari
@@ -9,6 +9,7 @@ type SignalDef struct {
|
||||
Signal string `json:"signal"` // upstream signal name (or "" for constant)
|
||||
Inputs []InputRef `json:"inputs"` // alternative multi-input format
|
||||
Pipeline []NodeDef `json:"pipeline"` // ordered list of DSP nodes
|
||||
Graph *Graph `json:"graph,omitempty"` // DAG form (preferred when present)
|
||||
Meta MetaOverride `json:"meta"` // optional metadata overrides
|
||||
|
||||
// Visibility controls who sees this signal in the signal tree:
|
||||
@@ -34,6 +35,37 @@ type NodeDef struct {
|
||||
Params map[string]any `json:"params,omitempty"`
|
||||
}
|
||||
|
||||
// Graph is the DAG form of a synthetic signal: a set of nodes (sources, ops and
|
||||
// one output) wired together by explicit per-node ordered input lists. It
|
||||
// supersedes the linear Inputs+Pipeline form: when SignalDef.Graph is set it is
|
||||
// authoritative; otherwise the legacy linear fields are converted into an
|
||||
// equivalent graph at load time (see toGraph).
|
||||
type Graph struct {
|
||||
Nodes []GraphNode `json:"nodes"`
|
||||
Output string `json:"output"` // id of the output node
|
||||
}
|
||||
|
||||
// GraphNode is one node in a Graph.
|
||||
//
|
||||
// kind=="source": carries DS+Signal; has no Inputs (a graph root).
|
||||
// kind=="op": carries Op + Params; Inputs lists upstream node IDs in the
|
||||
// order the op receives them (input 0, 1, … e.g. a−b, a÷b).
|
||||
// kind=="output": Inputs has a single upstream node whose value is the result.
|
||||
//
|
||||
// X/Y are the editor layout coordinates, persisted so a reloaded graph keeps its
|
||||
// shape; they have no effect on evaluation.
|
||||
type GraphNode struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Op string `json:"op,omitempty"`
|
||||
Params map[string]any `json:"params,omitempty"`
|
||||
DS string `json:"ds,omitempty"`
|
||||
Signal string `json:"signal,omitempty"`
|
||||
Inputs []string `json:"inputs,omitempty"`
|
||||
X float64 `json:"x,omitempty"`
|
||||
Y float64 `json:"y,omitempty"`
|
||||
}
|
||||
|
||||
// MetaOverride allows the synthetic signal to override display metadata.
|
||||
type MetaOverride struct {
|
||||
Unit string `json:"unit,omitempty"`
|
||||
|
||||
@@ -38,21 +38,32 @@ func stringParam(params map[string]any, key string) string {
|
||||
return s
|
||||
}
|
||||
|
||||
// BuildPipeline converts a []NodeDef (from JSON) into a []dsp.Node ready for
|
||||
// execution. JSON numbers are float64, so all numeric params are handled as
|
||||
// float64 regardless of the final type needed.
|
||||
func BuildPipeline(defs []NodeDef) ([]dsp.Node, error) {
|
||||
nodes := make([]dsp.Node, 0, len(defs))
|
||||
for i, d := range defs {
|
||||
n, err := buildNode(d)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pipeline node %d (%q): %w", i, d.Type, err)
|
||||
}
|
||||
nodes = append(nodes, n)
|
||||
// stringSliceParam extracts a []string from params; JSON arrays decode to []any,
|
||||
// so each element is coerced via its string value. Returns nil if missing.
|
||||
func stringSliceParam(params map[string]any, key string) []string {
|
||||
if params == nil {
|
||||
return nil
|
||||
}
|
||||
return nodes, nil
|
||||
v, ok := params[key]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
arr, ok := v.([]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(arr))
|
||||
for _, e := range arr {
|
||||
if s, ok := e.(string); ok && s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// buildNode converts a single NodeDef (from JSON) into a dsp.Node ready for
|
||||
// execution. JSON numbers are float64, so all numeric params are handled as
|
||||
// float64 regardless of the final type needed.
|
||||
func buildNode(d NodeDef) (dsp.Node, error) {
|
||||
p := d.Params
|
||||
switch d.Type {
|
||||
@@ -100,7 +111,7 @@ func buildNode(d NodeDef) (dsp.Node, error) {
|
||||
}, nil
|
||||
|
||||
case "expr":
|
||||
return &dsp.ExprNode{Expr: stringParam(p, "expr")}, nil
|
||||
return &dsp.ExprNode{Expr: stringParam(p, "expr"), Vars: stringSliceParam(p, "vars")}, nil
|
||||
|
||||
case "lowpass":
|
||||
order := int(floatParam(p, "order"))
|
||||
@@ -113,7 +124,7 @@ func buildNode(d NodeDef) (dsp.Node, error) {
|
||||
}, nil
|
||||
|
||||
case "lua":
|
||||
return &dsp.LuaNode{Script: stringParam(p, "script")}, nil
|
||||
return &dsp.LuaNode{Script: stringParam(p, "script"), Vars: stringSliceParam(p, "vars")}, nil
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown node type %q", d.Type)
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
package synthetic
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/uopi/uopi/internal/broker"
|
||||
"github.com/uopi/uopi/internal/dsp"
|
||||
)
|
||||
|
||||
// runtimeGraph is the executable form of a synthetic signal's DAG. Nodes are
|
||||
// held in topological order so a single forward pass computes every value with
|
||||
// each node's inputs already resolved. Op-node state maps persist across
|
||||
// evaluations (for stateful nodes like moving_average / lua).
|
||||
type runtimeGraph struct {
|
||||
order []*rtNode // topological order (sources first, output last)
|
||||
sources []rtSource // source nodes, in topological order
|
||||
outputID string // id of the output node
|
||||
}
|
||||
|
||||
type rtNode struct {
|
||||
id string
|
||||
kind string // source | op | output
|
||||
op dsp.Node // set for kind==op
|
||||
state map[string]any // persistent per-node state (op only)
|
||||
inputs []string // upstream node ids, in input order
|
||||
}
|
||||
|
||||
type rtSource struct {
|
||||
id string
|
||||
ref broker.SignalRef
|
||||
}
|
||||
|
||||
// sourceRefs returns the broker references for every source node, in a stable
|
||||
// order matching rg.sources.
|
||||
func (rg *runtimeGraph) sourceRefs() []broker.SignalRef {
|
||||
refs := make([]broker.SignalRef, len(rg.sources))
|
||||
for i, s := range rg.sources {
|
||||
refs[i] = s.ref
|
||||
}
|
||||
return refs
|
||||
}
|
||||
|
||||
// eval computes the output value given the latest value for each source node
|
||||
// (keyed by source node id). Nodes are visited in topological order so every
|
||||
// input is already present in vals by the time a node is processed.
|
||||
func (rg *runtimeGraph) eval(sourceVals map[string]float64) (float64, error) {
|
||||
vals := make(map[string]float64, len(rg.order))
|
||||
for id, v := range sourceVals {
|
||||
vals[id] = v
|
||||
}
|
||||
for _, n := range rg.order {
|
||||
switch n.kind {
|
||||
case "op":
|
||||
in := make([]float64, len(n.inputs))
|
||||
for i, id := range n.inputs {
|
||||
in[i] = vals[id]
|
||||
}
|
||||
r, err := n.op.Process(in, n.state)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("node %s (%s): %w", n.id, n.op.Type(), err)
|
||||
}
|
||||
vals[n.id] = r
|
||||
case "output":
|
||||
if len(n.inputs) > 0 {
|
||||
vals[n.id] = vals[n.inputs[0]]
|
||||
}
|
||||
}
|
||||
}
|
||||
return vals[rg.outputID], nil
|
||||
}
|
||||
|
||||
// compileGraph converts a SignalDef into an executable runtimeGraph. When the
|
||||
// def carries an explicit Graph it is used directly; otherwise the legacy
|
||||
// Inputs+Pipeline form is converted to an equivalent linear graph (see toGraph).
|
||||
func compileGraph(def SignalDef) (*runtimeGraph, error) {
|
||||
g := toGraph(def)
|
||||
if g == nil || len(g.Nodes) == 0 {
|
||||
return &runtimeGraph{}, nil
|
||||
}
|
||||
order, err := topoOrder(g)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rg := &runtimeGraph{outputID: g.Output}
|
||||
for _, gn := range order {
|
||||
switch gn.Kind {
|
||||
case "source":
|
||||
rg.sources = append(rg.sources, rtSource{id: gn.ID, ref: broker.SignalRef{DS: gn.DS, Name: gn.Signal}})
|
||||
case "op":
|
||||
node, err := buildNode(NodeDef{Type: gn.Op, Params: gn.Params})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("node %q: %w", gn.ID, err)
|
||||
}
|
||||
rg.order = append(rg.order, &rtNode{id: gn.ID, kind: "op", op: node, state: map[string]any{}, inputs: gn.Inputs})
|
||||
case "output":
|
||||
rg.outputID = gn.ID
|
||||
rg.order = append(rg.order, &rtNode{id: gn.ID, kind: "output", inputs: gn.Inputs})
|
||||
default:
|
||||
return nil, fmt.Errorf("node %q: unknown kind %q", gn.ID, gn.Kind)
|
||||
}
|
||||
}
|
||||
return rg, nil
|
||||
}
|
||||
|
||||
// topoOrder returns the graph's nodes in a topological (dependency-first) order,
|
||||
// treating each node's Inputs as its predecessors. It errors on dangling input
|
||||
// references or cycles.
|
||||
func topoOrder(g *Graph) ([]GraphNode, error) {
|
||||
byID := make(map[string]GraphNode, len(g.Nodes))
|
||||
for _, n := range g.Nodes {
|
||||
byID[n.ID] = n
|
||||
}
|
||||
indeg := make(map[string]int, len(g.Nodes))
|
||||
succ := make(map[string][]string, len(g.Nodes))
|
||||
for _, n := range g.Nodes {
|
||||
if _, ok := indeg[n.ID]; !ok {
|
||||
indeg[n.ID] = 0
|
||||
}
|
||||
for _, in := range n.Inputs {
|
||||
if _, ok := byID[in]; !ok {
|
||||
return nil, fmt.Errorf("node %q references unknown input %q", n.ID, in)
|
||||
}
|
||||
indeg[n.ID]++
|
||||
succ[in] = append(succ[in], n.ID)
|
||||
}
|
||||
}
|
||||
// Seed the queue with roots, preserving the node slice order for determinism.
|
||||
queue := make([]string, 0, len(g.Nodes))
|
||||
for _, n := range g.Nodes {
|
||||
if indeg[n.ID] == 0 {
|
||||
queue = append(queue, n.ID)
|
||||
}
|
||||
}
|
||||
order := make([]GraphNode, 0, len(g.Nodes))
|
||||
for len(queue) > 0 {
|
||||
id := queue[0]
|
||||
queue = queue[1:]
|
||||
order = append(order, byID[id])
|
||||
for _, s := range succ[id] {
|
||||
indeg[s]--
|
||||
if indeg[s] == 0 {
|
||||
queue = append(queue, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(order) != len(g.Nodes) {
|
||||
return nil, errors.New("graph contains a cycle")
|
||||
}
|
||||
return order, nil
|
||||
}
|
||||
|
||||
// toGraph returns the DAG for a SignalDef. If def.Graph is set it is returned
|
||||
// as-is. Otherwise the legacy linear form is converted: each input signal
|
||||
// becomes a source node, the pipeline becomes a chain of op nodes (the first op
|
||||
// receiving every source, each later op the previous op's output), terminated by
|
||||
// an output node. With no pipeline the output takes the first source directly,
|
||||
// matching the old runPipeline behaviour.
|
||||
func toGraph(def SignalDef) *Graph {
|
||||
if def.Graph != nil && len(def.Graph.Nodes) > 0 {
|
||||
return def.Graph
|
||||
}
|
||||
|
||||
inputs := def.Inputs
|
||||
if len(inputs) == 0 && def.DS != "" && def.Signal != "" {
|
||||
inputs = []InputRef{{DS: def.DS, Signal: def.Signal}}
|
||||
}
|
||||
|
||||
nodes := make([]GraphNode, 0, len(inputs)+len(def.Pipeline)+1)
|
||||
srcIDs := make([]string, 0, len(inputs))
|
||||
for i, inp := range inputs {
|
||||
id := fmt.Sprintf("s%d", i)
|
||||
nodes = append(nodes, GraphNode{ID: id, Kind: "source", DS: inp.DS, Signal: inp.Signal})
|
||||
srcIDs = append(srcIDs, id)
|
||||
}
|
||||
|
||||
opIDs := make([]string, 0, len(def.Pipeline))
|
||||
for i, nd := range def.Pipeline {
|
||||
id := fmt.Sprintf("p%d", i)
|
||||
var ins []string
|
||||
if i == 0 {
|
||||
ins = srcIDs
|
||||
} else {
|
||||
ins = []string{opIDs[i-1]}
|
||||
}
|
||||
nodes = append(nodes, GraphNode{ID: id, Kind: "op", Op: nd.Type, Params: nd.Params, Inputs: ins})
|
||||
opIDs = append(opIDs, id)
|
||||
}
|
||||
|
||||
var outInputs []string
|
||||
if len(opIDs) > 0 {
|
||||
outInputs = []string{opIDs[len(opIDs)-1]}
|
||||
} else if len(srcIDs) > 0 {
|
||||
outInputs = []string{srcIDs[0]}
|
||||
}
|
||||
nodes = append(nodes, GraphNode{ID: "out", Kind: "output", Inputs: outInputs})
|
||||
|
||||
return &Graph{Nodes: nodes, Output: "out"}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package synthetic
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// evalDef compiles a SignalDef and evaluates it against per-source values keyed
|
||||
// by source node id.
|
||||
func evalDef(t *testing.T, def SignalDef, srcVals map[string]float64) float64 {
|
||||
t.Helper()
|
||||
rg, err := compileGraph(def)
|
||||
if err != nil {
|
||||
t.Fatalf("compileGraph: %v", err)
|
||||
}
|
||||
out, err := rg.eval(srcVals)
|
||||
if err != nil {
|
||||
t.Fatalf("eval: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestGraphMultiInputDAG verifies that an intermediate op can take two
|
||||
// independently-wired sources — the capability the old linear pipeline lacked.
|
||||
func TestGraphMultiInputDAG(t *testing.T) {
|
||||
def := SignalDef{
|
||||
Name: "diff",
|
||||
Graph: &Graph{
|
||||
Output: "out",
|
||||
Nodes: []GraphNode{
|
||||
{ID: "a", Kind: "source", DS: "x", Signal: "left"},
|
||||
{ID: "b", Kind: "source", DS: "x", Signal: "right"},
|
||||
{ID: "sub", Kind: "op", Op: "subtract", Inputs: []string{"a", "b"}},
|
||||
{ID: "out", Kind: "output", Inputs: []string{"sub"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
got := evalDef(t, def, map[string]float64{"a": 10, "b": 3})
|
||||
if got != 7 {
|
||||
t.Errorf("subtract DAG: want 7, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGraphExprNamedInputs verifies expr nodes bind arbitrary named inputs in
|
||||
// wired order.
|
||||
func TestGraphExprNamedInputs(t *testing.T) {
|
||||
def := SignalDef{
|
||||
Name: "formula",
|
||||
Graph: &Graph{
|
||||
Output: "out",
|
||||
Nodes: []GraphNode{
|
||||
{ID: "a", Kind: "source", DS: "x", Signal: "p"},
|
||||
{ID: "b", Kind: "source", DS: "x", Signal: "q"},
|
||||
{ID: "e", Kind: "op", Op: "expr", Inputs: []string{"a", "b"},
|
||||
Params: map[string]any{"expr": "price * qty", "vars": []any{"price", "qty"}}},
|
||||
{ID: "out", Kind: "output", Inputs: []string{"e"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
got := evalDef(t, def, map[string]float64{"a": 4, "b": 2.5})
|
||||
if math.Abs(got-10) > 1e-9 {
|
||||
t.Errorf("expr named inputs: want 10, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGraphFanInToExpr exercises a non-trivial DAG: two ops feeding one expr.
|
||||
func TestGraphFanInToExpr(t *testing.T) {
|
||||
def := SignalDef{
|
||||
Name: "combo",
|
||||
Graph: &Graph{
|
||||
Output: "out",
|
||||
Nodes: []GraphNode{
|
||||
{ID: "a", Kind: "source", DS: "x", Signal: "p"},
|
||||
{ID: "b", Kind: "source", DS: "x", Signal: "q"},
|
||||
{ID: "g", Kind: "op", Op: "gain", Inputs: []string{"a"}, Params: map[string]any{"gain": 2.0}},
|
||||
{ID: "o", Kind: "op", Op: "offset", Inputs: []string{"b"}, Params: map[string]any{"offset": 1.0}},
|
||||
{ID: "e", Kind: "op", Op: "expr", Inputs: []string{"g", "o"}, Params: map[string]any{"expr": "a + b"}},
|
||||
{ID: "out", Kind: "output", Inputs: []string{"e"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
// g = 5*2 = 10 ; o = 4+1 = 5 ; a+b = 15
|
||||
got := evalDef(t, def, map[string]float64{"a": 5, "b": 4})
|
||||
if math.Abs(got-15) > 1e-9 {
|
||||
t.Errorf("fan-in DAG: want 15, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGraphLegacyConversion verifies the linear Inputs+Pipeline form still
|
||||
// evaluates correctly via the graph runtime.
|
||||
func TestGraphLegacyConversion(t *testing.T) {
|
||||
def := SignalDef{
|
||||
Name: "legacy",
|
||||
DS: "x",
|
||||
Signal: "p",
|
||||
Pipeline: []NodeDef{{Type: "gain", Params: map[string]any{"gain": 3.0}}},
|
||||
}
|
||||
rg, err := compileGraph(def)
|
||||
if err != nil {
|
||||
t.Fatalf("compileGraph: %v", err)
|
||||
}
|
||||
if len(rg.sources) != 1 {
|
||||
t.Fatalf("want 1 source, got %d", len(rg.sources))
|
||||
}
|
||||
got, err := rg.eval(map[string]float64{rg.sources[0].id: 4})
|
||||
if err != nil {
|
||||
t.Fatalf("eval: %v", err)
|
||||
}
|
||||
if got != 12 {
|
||||
t.Errorf("legacy gain: want 12, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGraphCycleRejected ensures a cyclic graph is refused at compile time.
|
||||
func TestGraphCycleRejected(t *testing.T) {
|
||||
def := SignalDef{
|
||||
Name: "cyclic",
|
||||
Graph: &Graph{
|
||||
Output: "out",
|
||||
Nodes: []GraphNode{
|
||||
{ID: "a", Kind: "op", Op: "gain", Inputs: []string{"b"}, Params: map[string]any{"gain": 1.0}},
|
||||
{ID: "b", Kind: "op", Op: "gain", Inputs: []string{"a"}, Params: map[string]any{"gain": 1.0}},
|
||||
{ID: "out", Kind: "output", Inputs: []string{"a"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
if _, err := compileGraph(def); err == nil {
|
||||
t.Error("expected cycle to be rejected")
|
||||
}
|
||||
}
|
||||
@@ -13,16 +13,14 @@ import (
|
||||
|
||||
"github.com/uopi/uopi/internal/broker"
|
||||
"github.com/uopi/uopi/internal/datasource"
|
||||
"github.com/uopi/uopi/internal/dsp"
|
||||
)
|
||||
|
||||
const definitionsFile = "synthetic.json"
|
||||
|
||||
// signalState holds everything needed to run one synthetic signal.
|
||||
type signalState struct {
|
||||
def SignalDef
|
||||
nodes []dsp.Node
|
||||
states []map[string]any // one map per node, persistent across calls
|
||||
def SignalDef
|
||||
rg *runtimeGraph // compiled DAG; op-node state persists across evaluations
|
||||
|
||||
// cancel stops the goroutine driving this signal.
|
||||
cancel context.CancelFunc
|
||||
@@ -123,19 +121,25 @@ func (s *Synthetic) Subscribe(ctx context.Context, signal string, ch chan<- data
|
||||
return nil, datasource.ErrNotFound
|
||||
}
|
||||
|
||||
// Collect the upstream references for this signal.
|
||||
refs := upstreamRefs(st.def)
|
||||
// Collect the source node references for this signal's DAG.
|
||||
refs := st.rg.sourceRefs()
|
||||
if len(refs) == 0 {
|
||||
return nil, fmt.Errorf("synthetic: signal %q has no upstream inputs", signal)
|
||||
}
|
||||
// Source node ids, index-aligned with refs, so updates map to graph inputs.
|
||||
srcIDs := make([]string, len(st.rg.sources))
|
||||
for i, s := range st.rg.sources {
|
||||
srcIDs[i] = s.id
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
|
||||
go func() {
|
||||
defer cancel()
|
||||
|
||||
// Latest value per upstream input index.
|
||||
latest := make([]float64, len(refs))
|
||||
// Latest value and timestamp per source node id.
|
||||
latest := make(map[string]float64, len(refs))
|
||||
latestTs := make([]time.Time, len(refs))
|
||||
ready := make([]bool, len(refs))
|
||||
|
||||
// Subscribe to every upstream ref via the broker.
|
||||
@@ -184,7 +188,8 @@ func (s *Synthetic) Subscribe(ctx context.Context, signal string, ch chan<- data
|
||||
return
|
||||
|
||||
case upd := <-updateCh:
|
||||
latest[upd.idx] = upd.val
|
||||
latest[srcIDs[upd.idx]] = upd.val
|
||||
latestTs[upd.idx] = upd.ts
|
||||
ready[upd.idx] = true
|
||||
|
||||
// Only compute once we have at least one value for every input.
|
||||
@@ -199,7 +204,19 @@ func (s *Synthetic) Subscribe(ctx context.Context, signal string, ch chan<- data
|
||||
continue
|
||||
}
|
||||
|
||||
// Run the pipeline.
|
||||
// The output is computed from the latest value of every input, so
|
||||
// its timestamp is the most recent contributing sample time. Using
|
||||
// the triggering update's timestamp instead would drag the output
|
||||
// back in time whenever a slow/stale input fired, producing
|
||||
// non-monotonic or duplicated timestamps on plots.
|
||||
outTs := latestTs[0]
|
||||
for _, ts := range latestTs[1:] {
|
||||
if ts.After(outTs) {
|
||||
outTs = ts
|
||||
}
|
||||
}
|
||||
|
||||
// Evaluate the DAG.
|
||||
s.mu.RLock()
|
||||
cur, stillExists := s.signals[signal]
|
||||
s.mu.RUnlock()
|
||||
@@ -207,14 +224,14 @@ func (s *Synthetic) Subscribe(ctx context.Context, signal string, ch chan<- data
|
||||
return
|
||||
}
|
||||
|
||||
result, err := runPipeline(cur.nodes, cur.states, latest)
|
||||
result, err := cur.rg.eval(latest)
|
||||
if err != nil {
|
||||
s.log.Warn("synthetic: pipeline error", "signal", signal, "err", err)
|
||||
continue
|
||||
}
|
||||
|
||||
v := datasource.Value{
|
||||
Timestamp: upd.ts,
|
||||
Timestamp: outTs,
|
||||
Data: result,
|
||||
Quality: datasource.QualityGood,
|
||||
}
|
||||
@@ -246,9 +263,9 @@ func (s *Synthetic) AddSignal(def SignalDef) error {
|
||||
return errors.New("signal name must not be empty")
|
||||
}
|
||||
|
||||
nodes, err := BuildPipeline(def.Pipeline)
|
||||
rg, err := compileGraph(def)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build pipeline: %w", err)
|
||||
return fmt.Errorf("compile graph: %w", err)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
@@ -257,16 +274,7 @@ func (s *Synthetic) AddSignal(def SignalDef) error {
|
||||
return fmt.Errorf("signal %q already exists", def.Name)
|
||||
}
|
||||
|
||||
states := make([]map[string]any, len(nodes))
|
||||
for i := range states {
|
||||
states[i] = make(map[string]any)
|
||||
}
|
||||
|
||||
st := &signalState{
|
||||
def: def,
|
||||
nodes: nodes,
|
||||
states: states,
|
||||
}
|
||||
st := &signalState{def: def, rg: rg}
|
||||
s.signals[def.Name] = st
|
||||
s.mu.Unlock()
|
||||
|
||||
@@ -320,14 +328,9 @@ func (s *Synthetic) UpdateSignal(def SignalDef) error {
|
||||
return errors.New("signal name must not be empty")
|
||||
}
|
||||
|
||||
nodes, err := BuildPipeline(def.Pipeline)
|
||||
rg, err := compileGraph(def)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build pipeline: %w", err)
|
||||
}
|
||||
|
||||
states := make([]map[string]any, len(nodes))
|
||||
for i := range states {
|
||||
states[i] = make(map[string]any)
|
||||
return fmt.Errorf("compile graph: %w", err)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
@@ -339,7 +342,7 @@ func (s *Synthetic) UpdateSignal(def SignalDef) error {
|
||||
if old.cancel != nil {
|
||||
old.cancel()
|
||||
}
|
||||
s.signals[def.Name] = &signalState{def: def, nodes: nodes, states: states}
|
||||
s.signals[def.Name] = &signalState{def: def, rg: rg}
|
||||
s.mu.Unlock()
|
||||
|
||||
if err := s.saveDefs(); err != nil {
|
||||
@@ -402,73 +405,22 @@ func (s *Synthetic) saveDefs() error {
|
||||
return os.WriteFile(s.defsFilePath(), data, 0o644)
|
||||
}
|
||||
|
||||
// startSignal builds the pipeline for def and registers the signalState.
|
||||
// startSignal compiles the DAG for def and registers the signalState.
|
||||
// The actual goroutines are started lazily by Subscribe.
|
||||
func (s *Synthetic) startSignal(def SignalDef) error {
|
||||
nodes, err := BuildPipeline(def.Pipeline)
|
||||
rg, err := compileGraph(def)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build pipeline for %q: %w", def.Name, err)
|
||||
}
|
||||
|
||||
states := make([]map[string]any, len(nodes))
|
||||
for i := range states {
|
||||
states[i] = make(map[string]any)
|
||||
return fmt.Errorf("compile graph for %q: %w", def.Name, err)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.signals[def.Name] = &signalState{
|
||||
def: def,
|
||||
nodes: nodes,
|
||||
states: states,
|
||||
}
|
||||
s.signals[def.Name] = &signalState{def: def, rg: rg}
|
||||
s.mu.Unlock()
|
||||
|
||||
s.log.Info("synthetic: signal registered", "name", def.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// runPipeline executes all nodes in sequence. The output of node N becomes
|
||||
// input[0] of node N+1. For the first node, inputs is the full upstream slice.
|
||||
func runPipeline(nodes []dsp.Node, states []map[string]any, inputs []float64) (float64, error) {
|
||||
if len(nodes) == 0 {
|
||||
if len(inputs) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return inputs[0], nil
|
||||
}
|
||||
|
||||
// First node receives all upstream inputs.
|
||||
cur := inputs
|
||||
var result float64
|
||||
var err error
|
||||
|
||||
for i, node := range nodes {
|
||||
result, err = node.Process(cur, states[i])
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("node %d (%s): %w", i, node.Type(), err)
|
||||
}
|
||||
// Subsequent nodes receive only the single output of the previous node.
|
||||
cur = []float64{result}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// upstreamRefs returns the broker.SignalRef list for a SignalDef.
|
||||
// If Inputs is set, those take precedence; otherwise DS+Signal is used.
|
||||
func upstreamRefs(def SignalDef) []broker.SignalRef {
|
||||
if len(def.Inputs) > 0 {
|
||||
refs := make([]broker.SignalRef, len(def.Inputs))
|
||||
for i, inp := range def.Inputs {
|
||||
refs[i] = broker.SignalRef{DS: inp.DS, Name: inp.Signal}
|
||||
}
|
||||
return refs
|
||||
}
|
||||
if def.DS != "" && def.Signal != "" {
|
||||
return []broker.SignalRef{{DS: def.DS, Name: def.Signal}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// defToMetadata converts a SignalDef into a datasource.Metadata.
|
||||
func defToMetadata(def SignalDef) datasource.Metadata {
|
||||
return datasource.Metadata{
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
package synthetic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/uopi/uopi/internal/broker"
|
||||
"github.com/uopi/uopi/internal/datasource"
|
||||
)
|
||||
|
||||
// seqSource is a test DataSource that emits a fixed sequence of values, each
|
||||
// carrying its own timestamp, so tests can control upstream sample times.
|
||||
type seqSource struct {
|
||||
name string
|
||||
seq []datasource.Value
|
||||
}
|
||||
|
||||
func (s *seqSource) Name() string { return s.name }
|
||||
func (s *seqSource) Connect(context.Context) error { return nil }
|
||||
func (s *seqSource) ListSignals(context.Context) ([]datasource.Metadata, error) { return nil, nil }
|
||||
func (s *seqSource) GetMetadata(context.Context, string) (datasource.Metadata, error) {
|
||||
return datasource.Metadata{Name: "x", Type: datasource.TypeFloat64}, nil
|
||||
}
|
||||
func (s *seqSource) Write(context.Context, string, any) error { return datasource.ErrNotWritable }
|
||||
func (s *seqSource) History(context.Context, string, time.Time, time.Time, int) ([]datasource.Value, error) {
|
||||
return nil, datasource.ErrHistoryUnavailable
|
||||
}
|
||||
func (s *seqSource) Subscribe(ctx context.Context, _ string, ch chan<- datasource.Value) (datasource.CancelFunc, error) {
|
||||
go func() {
|
||||
for _, v := range s.seq {
|
||||
select {
|
||||
case ch <- v:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
time.Sleep(8 * time.Millisecond)
|
||||
}
|
||||
}()
|
||||
return func() {}, nil
|
||||
}
|
||||
|
||||
// TestSubscribePreservesUpstreamTimestamp verifies a single-source synthetic
|
||||
// emits each computed value with the upstream sample's timestamp.
|
||||
func TestSubscribePreservesUpstreamTimestamp(t *testing.T) {
|
||||
log := slog.New(slog.NewTextHandler(os.Stderr, nil))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
base := time.Date(2026, 6, 19, 10, 0, 0, 0, time.UTC)
|
||||
src := &seqSource{name: "src", seq: []datasource.Value{
|
||||
{Timestamp: base.Add(1 * time.Second), Data: 1.0, Quality: datasource.QualityGood},
|
||||
{Timestamp: base.Add(2 * time.Second), Data: 2.0, Quality: datasource.QualityGood},
|
||||
{Timestamp: base.Add(3 * time.Second), Data: 3.0, Quality: datasource.QualityGood},
|
||||
}}
|
||||
|
||||
brk := broker.New(ctx, log)
|
||||
brk.Register(src)
|
||||
syn := New(t.TempDir(), brk, log)
|
||||
if err := syn.Connect(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := syn.AddSignal(SignalDef{
|
||||
Name: "g", DS: "src", Signal: "x",
|
||||
Pipeline: []NodeDef{{Type: "gain", Params: map[string]any{"gain": 10.0}}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ch := make(chan datasource.Value, 8)
|
||||
if _, err := syn.Subscribe(ctx, "g", ch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
want := []time.Time{base.Add(1 * time.Second), base.Add(2 * time.Second), base.Add(3 * time.Second)}
|
||||
for i, w := range want {
|
||||
select {
|
||||
case v := <-ch:
|
||||
if !v.Timestamp.Equal(w) {
|
||||
t.Errorf("emit #%d timestamp: want %s, got %s", i, w, v.Timestamp)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatalf("timeout waiting for emit #%d", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSubscribeMultiSourceUsesLatestTimestamp verifies that a synthetic combining
|
||||
// two independent sources stamps each output with the MOST RECENT contributing
|
||||
// sample time — not the timestamp of whichever source happened to trigger the
|
||||
// computation. A slow source carrying a stale timestamp must not drag the output
|
||||
// backwards in time (which previously produced wrong/non-monotonic plot points).
|
||||
func TestSubscribeMultiSourceUsesLatestTimestamp(t *testing.T) {
|
||||
log := slog.New(slog.NewTextHandler(os.Stderr, nil))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
now := time.Date(2026, 6, 19, 12, 0, 0, 0, time.UTC)
|
||||
// Fast source A with current timestamps.
|
||||
a := &seqSource{name: "A", seq: []datasource.Value{
|
||||
{Timestamp: now.Add(10 * time.Second), Data: 1.0, Quality: datasource.QualityGood},
|
||||
{Timestamp: now.Add(11 * time.Second), Data: 2.0, Quality: datasource.QualityGood},
|
||||
{Timestamp: now.Add(12 * time.Second), Data: 3.0, Quality: datasource.QualityGood},
|
||||
{Timestamp: now.Add(13 * time.Second), Data: 4.0, Quality: datasource.QualityGood},
|
||||
}}
|
||||
// Slow source B: a single sample with a much older timestamp.
|
||||
b := &seqSource{name: "B", seq: []datasource.Value{
|
||||
{Timestamp: now.Add(1 * time.Second), Data: 100.0, Quality: datasource.QualityGood},
|
||||
}}
|
||||
|
||||
brk := broker.New(ctx, log)
|
||||
brk.Register(a)
|
||||
brk.Register(b)
|
||||
syn := New(t.TempDir(), brk, log)
|
||||
if err := syn.Connect(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := syn.AddSignal(SignalDef{
|
||||
Name: "diff",
|
||||
Graph: &Graph{Output: "out", Nodes: []GraphNode{
|
||||
{ID: "sa", Kind: "source", DS: "A", Signal: "x"},
|
||||
{ID: "sb", Kind: "source", DS: "B", Signal: "x"},
|
||||
{ID: "sub", Kind: "op", Op: "subtract", Inputs: []string{"sa", "sb"}},
|
||||
{ID: "out", Kind: "output", Inputs: []string{"sub"}},
|
||||
}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ch := make(chan datasource.Value, 16)
|
||||
if _, err := syn.Subscribe(ctx, "diff", ch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var last time.Time
|
||||
for i := 0; i < 4; i++ {
|
||||
select {
|
||||
case v := <-ch:
|
||||
// The stale source-B timestamp (t=1s) must never be used: every output
|
||||
// is stamped with the newest input time, so emits stay monotonic.
|
||||
if v.Timestamp.Equal(now.Add(1 * time.Second)) {
|
||||
t.Errorf("emit #%d used the stale source-B timestamp %s", i, v.Timestamp)
|
||||
}
|
||||
if !last.IsZero() && v.Timestamp.Before(last) {
|
||||
t.Errorf("emit #%d went backwards: %s before previous %s", i, v.Timestamp, last)
|
||||
}
|
||||
last = v.Timestamp
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatalf("timeout waiting for emit #%d", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user