fixed and improved ui

2026-08-29 23:17:41 +02:00
parent 044ce57ba3
commit ec0a0cdb12
17 changed files with 997 additions and 198 deletions
@@ -1,7 +1,19 @@
 module marte2/common

-go 1.21
+go 1.24.9

 require github.com/gorilla/websocket v1.5.1

-require golang.org/x/net v0.17.0 // indirect
+require (
+	github.com/andybalholm/brotli v1.1.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/parquet-go/bitpack v1.0.0 // indirect
+	github.com/parquet-go/jsonlite v1.0.0 // indirect
+	github.com/parquet-go/parquet-go v0.32.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
+	github.com/twpayne/go-geom v1.6.1 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sys v0.38.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+)
@@ -1,4 +1,25 @@
+github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
+github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
 github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA=
+github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
+github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU=
+github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0=
+github.com/parquet-go/parquet-go v0.32.0 h1:NWDqTUHfrCS4cJP/Fj2HlxvqsrVedWG3sayMkf+znzM=
+github.com/parquet-go/parquet-go v0.32.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4=
+github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
@@ -422,6 +422,26 @@ func (hw *historyWriter) window() float64 {
 	return hw.windowSec
 }

+// coversWindow reports whether the archive file for key currently spans at
+// least sec seconds. When true, backfillCaptureHead can reconstruct a capture's
+// front out of the archive, so the trigger need not wait for the ring to cover
+// the whole window on its own.
+func (hw *historyWriter) coversWindow(key string, sec float64) bool {
+	if !hw.enabled() || !(sec > 0) {
+		return false
+	}
+	hw.mu.RLock()
+	hf, ok := hw.files[key]
+	hw.mu.RUnlock()
+	if !ok {
+		return false
+	}
+	hf.mu.RLock()
+	span := hf.tNewest - hf.tOldest
+	hf.mu.RUnlock()
+	return span >= sec
+}
+
 // setWindow points the archive at the timespan the clients are looking at, and
 // re-sizes the files that no longer match it. It reports whether any file's
 // geometry changed, which invalidates what clients know about the archive.
@@ -92,6 +92,14 @@ type triggerEngine struct {
 	bufGrowth float64
 	bufKnown  bool
 	bufRateOK bool
+	// bufCoverage is the maximum span (seconds) the ring can reach at its
+	// current bucket and capacity — the gate must never demand more than this,
+	// or a ring whose coverage is below the window can never satisfy it. 0 =
+	// unknown (no measurable rate).
+	bufCoverage float64
+	// bufArchived is true when the disk history already spans the trigger
+	// window, so a short capture's front can be back-filled from it.
+	bufArchived bool
 	// Reference point the growth is measured against.
 	bufRefSpan, bufRefWall float64

@@ -179,6 +187,7 @@ func (te *triggerEngine) SetConfig(cfg trigConfig) {
 	if base != te.baseKey {
 		// The buffer measurement belongs to the old signal's ring.
 		te.bufKnown, te.bufRateOK = false, false
+		te.bufCoverage, te.bufArchived = 0, false
 	}
 	te.baseKey, te.elemIdx = base, idx
 	te.prevValid = false
@@ -285,13 +294,16 @@ const bufGrowthIntervalSec = 0.5
 const bufGrowthSmooth = 0.5

 // setBuffered records how far back the trigger signal's ring reaches, at wall
-// clock now, and derives how fast that is growing. Pass known=false when there
-// is no such ring.
-func (te *triggerEngine) setBuffered(span float64, known bool, now float64) {
+// clock now, and derives how fast that is growing. coverage is the maximum
+// span (seconds) the ring can reach at its current bucket/capacity; archived
+// says the disk history already spans the trigger window. Pass known=false when
+// there is no ring to measure.
+func (te *triggerEngine) setBuffered(span, coverage float64, archived, known bool, now float64) {
 	te.mu.Lock()
 	defer te.mu.Unlock()
 	if !known {
 		te.bufKnown, te.bufRateOK = false, false
+		te.bufCoverage, te.bufArchived = 0, false
 		return
 	}
 	if !te.bufKnown {
@@ -299,6 +311,8 @@ func (te *triggerEngine) setBuffered(span float64, known bool, now float64) {
 		te.bufRefSpan, te.bufRefWall = span, now
 	}
 	te.bufSpan = span
+	te.bufCoverage = coverage
+	te.bufArchived = archived
 	dt := now - te.bufRefWall
 	if dt < bufGrowthIntervalSec {
 		return
@@ -336,9 +350,17 @@ func (te *triggerEngine) setBuffered(span float64, known bool, now float64) {
 // anyway. A full one grows only as fast as its incoming samples free space —
 // re-bucketing to a longer window replaces dense old samples with sparse new
 // ones — and it is that case, growth well below 1, where firing on the
-// pre-window alone delivers a capture whose front has been overwritten by the
-// time it is read. In the steady state growth is 0 and need is the whole
-// window, which a ring tuned for that window already exceeds, so nothing waits.
+//
+// Two escapes keep an armed trigger from staying deaf forever:
+//
+//   - archived  — the disk history already spans the window, so the front of a
+//     capture can be back-filled from it; the ring only needs to
+//     hold the pre-window worth of recent data.
+//   - coverage  — never demand more than the ring can physically reach. If its
+//     coverage saturates below the window (a measured source rate
+//     that over-estimates the true one), the gate opens once the
+//     ring is full anyway and a short capture is delivered instead
+//     of deafness.
 func (te *triggerEngine) fillNeedLocked() float64 {
 	pre := te.cfg.windowSec * te.cfg.prePercent / 100
 	growth := 0.0 // until measured, assume the buffer will not fill on its own
@@ -349,6 +371,14 @@ func (te *triggerEngine) fillNeedLocked() float64 {
 	if need < pre {
 		need = pre
 	}
+	if te.bufArchived {
+		// The archive back-fills the front; the ring holds the post-trigger
+		// window live, so the pre-window is all it needs to have reached.
+		return pre
+	}
+	if te.bufCoverage > 0 && need > te.bufCoverage {
+		need = te.bufCoverage
+	}
 	return need
 }

@@ -655,19 +685,32 @@ func (h *Hub) refreshTriggerFill() {
 		return
 	}
 	now := float64(time.Now().UnixNano()) / 1e9
+	key := h.trigger.baseSignalKey()
 	var rb *sigRing
-	if key := h.trigger.baseSignalKey(); key != "" {
+	if key != "" {
 		rb = h.getRing(key)
 	}
 	if rb == nil {
 		// Nothing to measure. Do not gate on a signal the hub does not carry:
 		// that would leave the trigger armed forever, which is worse than a
 		// short capture.
-		h.trigger.setBuffered(0, false, now)
+		h.trigger.setBuffered(0, 0, false, false, now)
 		return
 	}
 	_, span := rb.stats()
-	h.trigger.setBuffered(span, true, now)
+	// Maximum span the ring can ever reach at its current bucket/capacity, in
+	// seconds. The gate must never demand more than this, or a ring whose
+	// coverage is below the window (a measured source rate that over-estimates
+	// the true one) can never satisfy it.
+	coverage := 0.0
+	if rate := rb.sourceRate(); rate > 0 {
+		coverage = float64(ringCoverage(rb.bucketSize(), rb.capacity())) / rate
+	}
+	// If the disk archive already spans the trigger window, the front of a
+	// short capture can be back-filled from it, so the ring need not cover the
+	// whole window on its own.
+	archived := h.hist.coversWindow(key, h.trigger.Config().windowSec)
+	h.trigger.setBuffered(span, coverage, archived, true, now)
 }

 // triggerTick services the trigger FSM; called from Hub.Run() on every push tick.
@@ -123,7 +123,7 @@ func (s pulseTrainSim) run(t *testing.T, key string) pulseTrainResult {
 		tick := now + s.batchSec
 		h.retuneRings(tick)
 		_, span := h.rings[key].stats()
-		h.trigger.setBuffered(span, true, tick)
+		h.trigger.setBuffered(span, 0, false, true, tick)

 		if trigTime, pre, post, ok := h.trigger.dueCapture(tick); ok {
 			if buf := h.buildTriggerCapture(trigTime, pre, post); buf != nil {
@@ -297,9 +297,9 @@ func TestCollectingIsBroadcast(t *testing.T) {
 // later. It forgets any earlier measurement first, so the rate is the one
 // asked for rather than a blend with it.
 func setFill(te *triggerEngine, span, growth, now float64) {
-	te.setBuffered(0, false, now)
-	te.setBuffered(span-growth, true, now)
-	te.setBuffered(span, true, now+1)
+	te.setBuffered(0, 0, false, false, now)
+	te.setBuffered(span-growth, 0, false, true, now)
+	te.setBuffered(span, 0, false, true, now+1)
 }

 // What has to hold is that the buffer spans the whole window by the time the
@@ -417,9 +417,9 @@ func TestForceIgnoresFillGate(t *testing.T) {
 // interval, so they refresh the span and leave the seeded rate alone.
 func seedFillNow(te *triggerEngine, span, growth float64) {
 	now := float64(time.Now().UnixNano()) / 1e9
-	te.setBuffered(0, false, now-1)
-	te.setBuffered(span-growth, true, now-1)
-	te.setBuffered(span, true, now)
+	te.setBuffered(0, 0, false, false, now-1)
+	te.setBuffered(span-growth, 0, false, true, now-1)
+	te.setBuffered(span, 0, false, true, now)
 }

 // While it holds off, the trigger looks identical to one that is ignoring
@@ -505,3 +505,55 @@ func drainStates(t *testing.T, h *Hub) []map[string]any {
 		}
 	}
 }
+
+// A ring whose coverage saturates below the window (measured source rate that
+// over-estimates the true one) can never satisfy the full-window need. The
+// coverage clamp must open the gate once the ring is full, delivering a short
+// capture rather than staying deaf forever.
+func TestFillNeedClampedToCoverage(t *testing.T) {
+	te := newTriggerEngine()
+	te.SetConfig(trigConfig{signalKey: "s:x", windowSec: 60, prePercent: 20, mode: "normal", holdoffSec: 0.2})
+	setFill(te, 50, 0, 100) // ring full at 50 s, no growth
+	te.mu.Lock()
+	te.bufCoverage = 50 // the ring can never reach further back
+	te.mu.Unlock()
+
+	if need := te.fillNeedLocked(); need != 50 {
+		t.Errorf("need = %v, want 50 (clamped to coverage, not the 60 s window)", need)
+	}
+	if f := te.fillLocked(); f < 1 {
+		t.Errorf("fillLocked = %v, want >= 1: a full ring below the window must still open the gate", f)
+	}
+
+	// Without the clamp the gate would stay shut forever.
+	te.mu.Lock()
+	te.bufCoverage = 0
+	te.mu.Unlock()
+	if f := te.fillLocked(); f >= 1 {
+		t.Errorf("baseline: fillLocked = %v, want < 1 without a coverage clamp", f)
+	}
+}
+
+// When the disk archive already spans the window it can back-fill the front of
+// a capture, so the gate must only require the ring to have reached the
+// pre-window, not the whole window.
+func TestFillNeedArchiveLowersToPreWindow(t *testing.T) {
+	te := newTriggerEngine()
+	te.SetConfig(trigConfig{signalKey: "s:x", windowSec: 60, prePercent: 20, mode: "normal", holdoffSec: 0.2})
+	setFill(te, 30, 0, 100) // ring holds only 30 s, no growth → need 60 without archive
+	te.mu.Lock()
+	te.bufArchived = true
+	te.mu.Unlock()
+
+	if want := 12.0; te.fillNeedLocked() != want { // 60 * 0.20
+		t.Errorf("need = %v, want %v (archive lowers to the pre-window)", te.fillNeedLocked(), want)
+	}
+
+	// A ring holding just the pre-window opens the gate once archived.
+	te.mu.Lock()
+	te.bufSpan = 12
+	te.mu.Unlock()
+	if f := te.fillLocked(); f < 1 {
+		t.Errorf("fillLocked = %v, want >= 1 with pre-window buffered and the archive available", f)
+	}
+}