Implemented and fixed many issues

2026-08-21 23:24:48 +02:00
parent 14d5351a81
commit e03c60db25
52 changed files with 7726 additions and 769 deletions
@@ -3,7 +3,9 @@ package wshub
 import (
 	"encoding/binary"
 	"encoding/json"
+	"log"
 	"math"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -25,10 +27,30 @@ const (
 // capture is extracted, so the rings have received the last samples.
 const captureMarginSec = 0.15

+// captureStallSec is how long the stream may be silent before a collecting
+// trigger gives up waiting for the rest of its window and delivers what it has.
+const captureStallSec = 2.0
+
 // autoRearmDelaySec is the pause between a completed capture and the automatic
 // rearm in "normal" mode.
 const autoRearmDelaySec = 0.2

+// trigCapturePts caps the points sent per signal in a capture frame. A window
+// of 60 s at 1 MSps is 60 M raw samples — ~960 MB per signal on the wire, which
+// no client can take and which the send path would simply drop. Matches the C++
+// StreamHub's kTrigCapturePts.
+const trigCapturePts = 20000
+
+// shortCaptureTol is the fraction of the window a capture may miss at its front
+// before it is reported. One min/max bucket of slack, not a quality target.
+const shortCaptureTol = 0.01
+
+// maxTriggerWindowSec bounds the capture window, matching the longest option
+// the web UI offers. It is not a resolution limit: retuneRings buckets the
+// rings so any window fits the per-signal memory budget, at the cost of storing
+// min/max pairs rather than every sample.
+const maxTriggerWindowSec = 600.0
+
 // trigConfig is the client-settable part of the trigger.
 type trigConfig struct {
 	signalKey  string // "src:sig" or "src:sig[i]"
@@ -36,7 +58,8 @@ type trigConfig struct {
 	threshold  float64
 	windowSec  float64
 	prePercent float64
-	mode       string // "normal" | "single"
+	mode       string  // "normal" | "single"
+	holdoffSec float64 // rearm delay after a capture (double-trigger guard)
 }

 // triggerEngine implements the hub-side trigger FSM. Its methods are safe to
@@ -51,11 +74,34 @@ type triggerEngine struct {

 	state   string
 	stopped bool
+	// sentState is the state carried by the last stateMsg handed out. The
+	// armed→collecting transition happens inside feed(), on the ingest path,
+	// so the hub cannot see it by sampling State() across a tick — by the time
+	// the tick runs, ingest has already moved the FSM.
+	sentState string
+	// sentFill is the pre-fill fraction carried by the last stateMsg, so a
+	// trigger that is armed but still filling can report progress.
+	sentFill float64
+
+	// How far back the trigger signal's ring reaches and how fast that is
+	// growing (seconds of span per second of wall clock), refreshed by the hub.
+	// bufKnown is false when there is no ring to measure, which disables the
+	// fill gate rather than blocking the trigger on a measurement that will
+	// never arrive; bufRateOK is false until two measurements exist.
+	bufSpan   float64
+	bufGrowth float64
+	bufKnown  bool
+	bufRateOK bool
+	// Reference point the growth is measured against.
+	bufRefSpan, bufRefWall float64

 	prevValue float64
 	prevValid bool
 	lastT     float64
 	lastTOK   bool
+	// lastFeedWall is the wall clock at the last feed(), used only to notice a
+	// stalled stream — the window itself is measured on the sample clock.
+	lastFeedWall float64

 	trigTime   float64
 	firedPre   float64
@@ -67,7 +113,7 @@ type triggerEngine struct {

 func newTriggerEngine() *triggerEngine {
 	return &triggerEngine{
-		cfg:     trigConfig{edge: "rising", windowSec: 1, prePercent: 20, mode: "normal"},
+		cfg:     trigConfig{edge: "rising", windowSec: 1, prePercent: 20, mode: "normal", holdoffSec: autoRearmDelaySec},
 		elemIdx: -1,
 		state:   trigIdle,
 	}
@@ -97,8 +143,8 @@ func (te *triggerEngine) SetConfig(cfg trigConfig) {
 	if cfg.windowSec < 1e-4 {
 		cfg.windowSec = 1e-4
 	}
-	if cfg.windowSec > 10 {
-		cfg.windowSec = 10
+	if cfg.windowSec > maxTriggerWindowSec {
+		cfg.windowSec = maxTriggerWindowSec
 	}
 	if cfg.prePercent < 0 {
 		cfg.prePercent = 0
@@ -106,8 +152,19 @@ func (te *triggerEngine) SetConfig(cfg trigConfig) {
 	if cfg.prePercent > 100 {
 		cfg.prePercent = 100
 	}
+	if cfg.holdoffSec < 0 {
+		cfg.holdoffSec = 0
+	}
+	if cfg.holdoffSec > 60 {
+		cfg.holdoffSec = 60
+	}
 	te.cfg = cfg
-	te.baseKey, te.elemIdx = parseSignalKey(cfg.signalKey)
+	base, idx := parseSignalKey(cfg.signalKey)
+	if base != te.baseKey {
+		// The buffer measurement belongs to the old signal's ring.
+		te.bufKnown, te.bufRateOK = false, false
+	}
+	te.baseKey, te.elemIdx = base, idx
 	te.prevValid = false
 	te.prevValue = 0
 }
@@ -168,6 +225,105 @@ func (te *triggerEngine) Active() bool {
 	return te.baseKey != ""
 }

+// baseSignalKey is the configured trigger signal without its "[i]" suffix, or
+// "" when no trigger signal is set.
+func (te *triggerEngine) baseSignalKey() string {
+	te.mu.Lock()
+	defer te.mu.Unlock()
+	return te.baseKey
+}
+
+// bufGrowthIntervalSec is the shortest baseline the span growth is measured
+// over. The hub refreshes 30 times a second and the span moves in steps as
+// batches land, so a shorter baseline measures the batching, not the trend.
+const bufGrowthIntervalSec = 0.5
+
+// bufGrowthSmooth is the weight of a new growth measurement in the running
+// estimate.
+const bufGrowthSmooth = 0.5
+
+// setBuffered records how far back the trigger signal's ring reaches, at wall
+// clock now, and derives how fast that is growing. Pass known=false when there
+// is no such ring.
+func (te *triggerEngine) setBuffered(span float64, known bool, now float64) {
+	te.mu.Lock()
+	defer te.mu.Unlock()
+	if !known {
+		te.bufKnown, te.bufRateOK = false, false
+		return
+	}
+	if !te.bufKnown {
+		te.bufKnown = true
+		te.bufRefSpan, te.bufRefWall = span, now
+	}
+	te.bufSpan = span
+	dt := now - te.bufRefWall
+	if dt < bufGrowthIntervalSec {
+		return
+	}
+	g := (span - te.bufRefSpan) / dt
+	// A ring that is not full grows one second of span per second; one that is
+	// full grows by whatever its incoming samples free up. Neither can exceed 1,
+	// and a shrinking ring is simply not growing.
+	if g < 0 {
+		g = 0
+	} else if g > 1 {
+		g = 1
+	}
+	if te.bufRateOK {
+		g = te.bufGrowth + bufGrowthSmooth*(g-te.bufGrowth)
+	}
+	te.bufGrowth, te.bufRateOK = g, true
+	te.bufRefSpan, te.bufRefWall = span, now
+}
+
+// fillNeedLocked is how far back the buffer must reach before an edge may be
+// accepted, so that the capture is still whole when it is harvested a
+// post-window later.
+//
+// What has to hold at harvest time is that the buffer spans the whole window:
+// its newest sample is then trigTime+post, so anything less has lost the front
+// of the capture. The buffer keeps filling while the post-window is collected,
+// though, so the shortfall it may start with is exactly what it will make up in
+// that time — measured, not assumed:
+//
+//	need = windowSec − growth × postSec, floored at the pre-trigger window
+//
+// A ring that is still filling grows a second per second, which reduces this to
+// the pre-trigger window: everything after the trigger is yet to be recorded
+// anyway. A full one grows only as fast as its incoming samples free space —
+// re-bucketing to a longer window replaces dense old samples with sparse new
+// ones — and it is that case, growth well below 1, where firing on the
+// pre-window alone delivers a capture whose front has been overwritten by the
+// time it is read. In the steady state growth is 0 and need is the whole
+// window, which a ring tuned for that window already exceeds, so nothing waits.
+func (te *triggerEngine) fillNeedLocked() float64 {
+	pre := te.cfg.windowSec * te.cfg.prePercent / 100
+	growth := 0.0 // until measured, assume the buffer will not fill on its own
+	if te.bufRateOK {
+		growth = te.bufGrowth
+	}
+	need := te.cfg.windowSec - growth*(te.cfg.windowSec-pre)
+	if need < pre {
+		need = pre
+	}
+	return need
+}
+
+// fillLocked is how much of that requirement is met, as a fraction in [0, 1].
+// It is 1 whenever the gate does not apply: nothing needed, or no ring to
+// measure.
+func (te *triggerEngine) fillLocked() float64 {
+	need := te.fillNeedLocked()
+	if need <= 0 || !te.bufKnown || te.bufSpan >= need*(1-shortCaptureTol) {
+		return 1
+	}
+	if te.bufSpan <= 0 {
+		return 0
+	}
+	return te.bufSpan / need
+}
+
 // latchWindowLocked freezes the pre/post split at fire time so later config
 // edits do not change how the capture is rendered.
 func (te *triggerEngine) latchWindowLocked(t float64) {
@@ -209,6 +365,7 @@ func (te *triggerEngine) feed(key string, nElem int, t, v []float64) {
 	}
 	te.lastT = t[len(t)-1]
 	te.lastTOK = true
+	te.lastFeedWall = float64(time.Now().UnixNano()) / 1e9
 	if te.state != trigArmed {
 		return
 	}
@@ -219,6 +376,17 @@ func (te *triggerEngine) feed(key string, nElem int, t, v []float64) {
 		}
 		step, start = nElem, te.elemIdx
 	}
+	// Hold off while the buffer does not reach back far enough. Firing now would
+	// deliver a capture whose front is simply missing — the ring never held it —
+	// which is what made the first shot after a window change come back short.
+	// Track the level meanwhile, so the first edge once the buffer is deep
+	// enough is still measured against the right previous sample.
+	if te.fillLocked() < 1 {
+		for i := start; i < len(v); i += step {
+			te.prevValue, te.prevValid = v[i], true
+		}
+		return
+	}
 	thr := te.cfg.threshold
 	for i := start; i < len(t); i += step {
 		if !te.prevValid {
@@ -247,13 +415,28 @@ func (te *triggerEngine) feed(key string, nElem int, t, v []float64) {

 // dueCapture reports whether a collecting trigger's post-window has elapsed and
 // returns the latched window.
+//
+// The window is measured on the sample clock, not the wall clock: trigTime is a
+// sample timestamp, and a stream whose timestamps lag real time (a busy
+// producer, a buffered link) would otherwise be cut short by exactly that lag —
+// an 8 s lag turned a 60 s window into a 36 s capture. Waiting for the samples
+// themselves also means the ring really holds the window by the time it is read.
 func (te *triggerEngine) dueCapture(nowSec float64) (trigTime, pre, post float64, ok bool) {
 	te.mu.Lock()
 	defer te.mu.Unlock()
 	if te.state != trigCollecting || !te.firedValid {
 		return 0, 0, 0, false
 	}
-	if nowSec < te.trigTime+te.firedPost+captureMarginSec {
+	deadline := te.trigTime + te.firedPost + captureMarginSec
+	switch {
+	case te.lastTOK && te.lastT >= deadline:
+	// The samples have covered the window.
+	case !te.lastTOK && nowSec >= deadline:
+		// No sample ever seen, so trigTime came from the wall clock (Force).
+	case te.lastFeedWall > 0 && nowSec-te.lastFeedWall >= captureStallSec:
+		// The stream has dried up; deliver what was collected rather than
+		// leaving the client stuck in "collecting" forever.
+	default:
 		return 0, 0, 0, false
 	}
 	return te.trigTime, te.firedPre, te.firedPost, true
@@ -266,7 +449,7 @@ func (te *triggerEngine) markTriggered(nowSec float64) {
 	if te.state == trigCollecting {
 		te.state = trigTriggered
 		if te.cfg.mode != "single" && !te.stopped {
-			te.rearmAt = nowSec + autoRearmDelaySec
+			te.rearmAt = nowSec + te.cfg.holdoffSec
 		}
 	}
 	te.mu.Unlock()
@@ -283,17 +466,49 @@ func (te *triggerEngine) dueRearm(nowSec float64) bool {
 	return !te.stopped
 }

+// stateUnsent reports whether the FSM has moved since the last stateMsg was
+// built, i.e. whether clients still have to be told.
+func (te *triggerEngine) stateUnsent() bool {
+	te.mu.Lock()
+	defer te.mu.Unlock()
+	if te.state != te.sentState {
+		return true
+	}
+	// An armed trigger waiting for its buffer is otherwise indistinguishable
+	// from one that is ignoring edges, so the filling itself is news. Coarse
+	// steps only: this is checked 30 times a second.
+	if te.state == trigArmed {
+		f := te.fillLocked()
+		return math.Abs(f-te.sentFill) >= 0.02 || (f >= 1 && te.sentFill < 1)
+	}
+	return false
+}
+
 // stateMsg builds the JSON "triggerState" broadcast for the current FSM state.
 func (te *triggerEngine) stateMsg() []byte {
 	te.mu.Lock()
+	te.sentState = te.state
+	te.sentFill = te.fillLocked()
 	m := map[string]any{
 		"type":    "triggerState",
 		"state":   te.state,
 		"mode":    te.cfg.mode,
 		"stopped": te.stopped,
 	}
+	if te.state == trigArmed && te.sentFill < 1 {
+		// Armed but holding off: the buffer does not yet reach back far enough
+		// to deliver the window, so edges are being ignored on purpose.
+		m["bufferFill"] = te.sentFill
+		m["bufferNeedSec"] = te.fillNeedLocked()
+	}
 	if te.firedValid {
+		// The window latched at fire time. Clients draw the filling capture on
+		// this axis before the v2 frame arrives, and config edits between arm
+		// and fire would otherwise leave them inferring the wrong window from
+		// their own copy of the config.
 		m["trigTime"] = te.trigTime
+		m["preSec"] = te.firedPre
+		m["postSec"] = te.firedPost
 	}
 	te.mu.Unlock()
 	msg, _ := json.Marshal(m)
@@ -331,6 +546,9 @@ func (h *Hub) handleTriggerCommand(t string, env map[string]interface{}) bool {
 		if f, ok := env["prePercent"].(float64); ok {
 			cfg.prePercent = f
 		}
+		if f, ok := env["holdoffSec"].(float64); ok {
+			cfg.holdoffSec = f
+		}
 		h.trigger.SetConfig(cfg)
 	case "arm", "rearm":
 		h.trigger.Arm()
@@ -347,34 +565,139 @@ func (h *Hub) handleTriggerCommand(t string, env map[string]interface{}) bool {
 	default:
 		return false
 	}
+	// Measure the buffer now rather than waiting for the next tick: ingest runs
+	// on the source goroutine and a 1 MSps stream crosses the threshold many
+	// times within one 33 ms tick, so an arm serviced here would otherwise fire
+	// on a stale (or missing) measurement before the gate ever saw the new
+	// configuration.
+	h.refreshTriggerFill()
 	h.broadcastTriggerState()
 	return true
 }

+// refreshTriggerFill tells the FSM how far back the trigger signal's ring
+// reaches, which is what lets an armed trigger hold off until a capture taken
+// now would come back whole.
+//
+// The ring is the right yardstick even though a short capture is back-filled
+// from the archive: the archive is sized for the same window and starts over
+// whenever that window changes, so it holds no more of the stretch being waited
+// for than the ring does. It can only add to what the capture finds.
+//
+// Called both from the push tick and from the client goroutine handling a
+// trigger command; all the state it derives lives in the engine, behind the
+// engine's lock.
+func (h *Hub) refreshTriggerFill() {
+	if h.trigger == nil {
+		return
+	}
+	now := float64(time.Now().UnixNano()) / 1e9
+	var rb *sigRing
+	if key := h.trigger.baseSignalKey(); key != "" {
+		rb = h.getRing(key)
+	}
+	if rb == nil {
+		// Nothing to measure. Do not gate on a signal the hub does not carry:
+		// that would leave the trigger armed forever, which is worse than a
+		// short capture.
+		h.trigger.setBuffered(0, false, now)
+		return
+	}
+	_, span := rb.stats()
+	h.trigger.setBuffered(span, true, now)
+}
+
 // triggerTick services the trigger FSM; called from Hub.Run() on every push tick.
 func (h *Hub) triggerTick() {
 	nowSec := float64(time.Now().UnixNano()) / 1e9
-	prev := h.trigger.State()
+
+	h.retuneRings(nowSec)
+	h.openPendingHistoryFiles(nowSec)
+	h.refreshTriggerFill()

 	if trigTime, pre, post, ok := h.trigger.dueCapture(nowSec); ok {
 		if msg := h.buildTriggerCapture(trigTime, pre, post); msg != nil {
+			dropped := 0
 			for c := range h.clients {
 				select {
 				case c.send <- wsMessage{websocket.BinaryMessage, msg}:
 				default:
+					dropped++
 				}
 			}
+			// A dropped capture is invisible to the user — the trigger fires,
+			// the state goes to "triggered" and no waveform ever arrives — so
+			// say so rather than leaving it to be guessed at.
+			if dropped > 0 {
+				log.Printf("wshub: trigger capture (%d B) dropped for %d client(s): send queue full",
+					len(msg), dropped)
+			}
 		}
 		h.trigger.markTriggered(nowSec)
+		// A capture is only zoomable for as long as its samples still exist at
+		// full resolution somewhere, and the rings roll past the window within
+		// seconds of it being taken. Lift the window out of the archive into a
+		// file of its own, where nothing overwrites it until the next trigger.
+		h.hist.captureRange(trigTime-pre, trigTime+post)
 	} else if h.trigger.dueRearm(nowSec) {
 		h.trigger.Arm()
 	}

-	if h.trigger.State() != prev {
+	if h.trigger.stateUnsent() {
 		h.broadcastTriggerState()
 	}
 }

+// backfillCaptureHead prepends the front of [t0, t1] that the ring no longer
+// holds, read from the disk archive. It returns its input unchanged when the
+// ring already reaches t0, when history is off, or when the archive has nothing
+// for that range.
+//
+// The rings are sized for the window, but they only have to *become* that long:
+// they are min/max buckets that cover the configured window once they have
+// rolled over completely at the current bucket, which takes as long as the
+// window itself. Widen the window and arm, and the first captures ask for more
+// history than the ring has ever stored — the frame then starts late and the
+// user sees a blank front half. The archive is written straight through, at the
+// geometry its file was created with, so unless that file was re-sized too it
+// has kept the stretch the ring is still converging on.
+func (h *Hub) backfillCaptureHead(key string, t0, t1 float64, st, sv []float64) ([]float64, []float64) {
+	window := t1 - t0
+	if !h.hist.enabled() || window <= 0 {
+		return st, sv
+	}
+	gapEnd := t1
+	if len(st) > 0 {
+		gapEnd = st[0]
+	}
+	gap := gapEnd - t0
+	if gap <= shortCaptureTol*window {
+		return st, sv
+	}
+	// Budget the read by the share of the window being back-filled. The frame is
+	// decimated to trigCapturePts either way, so a bigger read would buy nothing
+	// but disk seeks — on the hub's own goroutine, between two push ticks.
+	maxOut := int(float64(trigCapturePts)*gap/window) + 2
+	ht, hv := h.hist.readRange(key, t0, gapEnd, maxOut)
+	if len(ht) == 0 {
+		return st, sv
+	}
+	// Drop anything at or past the ring's first sample: the two sources overlap
+	// around the join, and the frame's timestamps must stay ascending.
+	n := len(ht)
+	if len(st) > 0 {
+		n = sort.SearchFloat64s(ht, st[0])
+	}
+	if n == 0 {
+		return st, sv
+	}
+	outT := make([]float64, 0, n+len(st))
+	outV := make([]float64, 0, n+len(sv))
+	outT = append(append(outT, ht[:n]...), st...)
+	outV = append(append(outV, hv[:n]...), sv...)
+	return outT, outV
+}
+
 // buildTriggerCapture extracts [trigTime-pre, trigTime+post] from every ring
 // buffer and encodes the version-2 binary capture frame:
 //
@@ -397,18 +720,41 @@ func (h *Hub) buildTriggerCapture(trigTime, pre, post float64) []byte {
 	h.ringsMu.RUnlock()

 	slices := make([]sigSlice, 0, len(keys))
+	held := make(map[string]sigData, len(keys))
 	total := 1 + 8 + 8 + 8 + 4
 	for i, k := range keys {
 		st, sv := rings[i].slice(t0, t1)
+		st, sv = h.backfillCaptureHead(k, t0, t1, st, sv)
 		if len(st) == 0 {
 			continue
 		}
+		// Neither the ring nor the archive reached t0. Nothing can recover that
+		// data, so name it rather than leaving the user to wonder why the front
+		// of their window is blank.
+		if lost := st[0] - t0; lost > shortCaptureTol*(t1-t0) {
+			cnt, span := rings[i].stats()
+			log.Printf("wshub: capture %s is short by %.2f s of %.2f s: ring holds %.2f s (%d pts, min/max over %d)",
+				k, lost, t1-t0, span, cnt, rings[i].bucketSize())
+		}
+		// Take the second half of the double buffer here, before the frame is
+		// decimated: the client gets 20 000 points to draw, but a zoom into
+		// them has to come back with the underlying samples, and the rings will
+		// have rolled past them by the time it is asked for.
+		held[k] = sigData{T: st, V: sv}
+		// Decimate before framing: a long window at a high sample rate is
+		// hundreds of megabytes raw, which the send path would silently drop.
+		// The min/max envelope keeps every peak in the window, so a glitch is
+		// still on screen at the zoomed-out view that first shows it.
+		st, sv = minMaxDecimate(st, sv, trigCapturePts)
 		slices = append(slices, sigSlice{key: k, t: st, v: sv})
 		total += 2 + len(k) + 4 + len(st)*16
 	}
 	if len(slices) == 0 {
 		return nil
 	}
+	// Swap only now that the capture is known good. A shot that yielded nothing
+	// must leave the previous window on screen rather than blanking it.
+	h.capture.publish(t0, t1, held)

 	buf := make([]byte, total)
 	buf[0] = 2