Martino Ferrari
356 lines
15 KiB
Python
356 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
validate_waveform.py — Per-effect waveform validator for the streaming-chain E2E.
|
|
|
|
Compares the client-recorded stream (``received_<id>.bin``, RCV1 format written
|
|
by chain-client) against the analytic ground truth (rebuilt from gen_data) and,
|
|
when present, the fed-reference tap (``tap_<id>.bin``, MARTe binary). It also
|
|
rolls in the client behavioural checks (``checks_<id>.json``) and emits
|
|
``metrics_<id>.json`` with an overall pass/fail (exit 0/1).
|
|
|
|
Oracle (per signal)
|
|
-------------------
|
|
* **Fidelity** (always): every received value must lie within ``tol`` of *some*
|
|
ground-truth value. ``tol`` is 0 (bit-exact) for un-quantised integers, a tiny
|
|
float epsilon for un-quantised floats, and ``quant_step/2 + 1e-6·range`` for
|
|
quantised floats. Catches type corruption and out-of-range quantisation.
|
|
* **Shape** (sine signals, ≥8 points): least-squares fit of
|
|
``a·sin(ωt)+b·cos(ωt)+c`` at the scenario frequency ω=2πf. A high correlation
|
|
(≥0.99) and low normalised RMSE confirm the received waveform is the expected
|
|
sinusoid at the expected frequency (the fit recovers the unknown wall-clock
|
|
phase offset automatically). nRMSE tolerance is relaxed by the quant step.
|
|
* **Fed reference** (when ``--tap`` given): each received value must be within
|
|
``tol`` of some tap value too.
|
|
* **Continuity** (always, ≥10 points): a stream that stalls (client falling
|
|
behind, hub failing to flush a window, ...) can still pass fidelity —
|
|
whatever few samples *did* arrive still match the ground truth — while the
|
|
plot shows gaping holes. Flags any inter-sample gaps that are >10x the
|
|
median spacing and fails when their *summed* duration exceeds 5% of the
|
|
capture span. Calibrated against the full scenario matrix: healthy streams
|
|
(bursty per-tick live pushes, decimation, fragmentation, multicast, ...) top
|
|
out at 0.7% outlier-gap time; a stalled stream showed 55-91%.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import struct
|
|
import sys
|
|
|
|
import numpy as np
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import scenarios as S # noqa: E402
|
|
import gen_data as G # noqa: E402
|
|
|
|
CODE_TO_TYPE = {v: k for k, v in S.TYPE_CODES.items()}
|
|
|
|
|
|
# ── readers ──────────────────────────────────────────────────────────────────
|
|
|
|
def read_received(path):
|
|
"""Read RCV1 → {key: (t ndarray, v ndarray)}."""
|
|
with open(path, "rb") as f:
|
|
data = f.read()
|
|
if data[:4] != b"RCV1":
|
|
raise ValueError(f"{path}: bad magic")
|
|
off = 4
|
|
nsig = struct.unpack_from("<I", data, off)[0]
|
|
off += 4
|
|
out = {}
|
|
for _ in range(nsig):
|
|
klen = struct.unpack_from("<H", data, off)[0]
|
|
off += 2
|
|
key = data[off:off + klen].decode()
|
|
off += klen
|
|
n = struct.unpack_from("<I", data, off)[0]
|
|
off += 4
|
|
t = np.frombuffer(data, "<f8", n, off).copy()
|
|
off += 8 * n
|
|
v = np.frombuffer(data, "<f8", n, off).copy()
|
|
off += 8 * n
|
|
out[key] = (t, v)
|
|
return out
|
|
|
|
|
|
def read_marte_binary(path):
|
|
"""Read a MARTe binary file → {name: ndarray[n_rows, elements]} (typed)."""
|
|
with open(path, "rb") as f:
|
|
data = f.read()
|
|
ns = struct.unpack_from("<I", data, 0)[0]
|
|
off = 4
|
|
descs = []
|
|
for _ in range(ns):
|
|
tc = struct.unpack_from("<H", data, off)[0]
|
|
name = data[off + 2:off + 34].rstrip(b"\0").decode(errors="replace")
|
|
ne = struct.unpack_from("<I", data, off + 34)[0]
|
|
descs.append((name, tc, ne))
|
|
off += 38
|
|
row_bytes = sum(struct.calcsize(_npfmt(tc)) * ne for _, tc, ne in descs)
|
|
nrows = (len(data) - off) // row_bytes if row_bytes else 0
|
|
cols = {name: [] for name, _, _ in descs}
|
|
for r in range(nrows):
|
|
for name, tc, ne in descs:
|
|
dt = np.dtype(S.NP_DTYPE[CODE_TO_TYPE[tc]])
|
|
vals = np.frombuffer(data, dt, ne, off)
|
|
cols[name].append(vals.astype(np.float64))
|
|
off += dt.itemsize * ne
|
|
return {name: np.array(cols[name]) for name, _, _ in descs}
|
|
|
|
|
|
def _npfmt(tc):
|
|
return {"<i1": "b", "<u1": "B", "<i2": "h", "<u2": "H", "<i4": "i",
|
|
"<u4": "I", "<i8": "q", "<u8": "Q", "<f4": "f",
|
|
"<f8": "d"}[S.NP_DTYPE[CODE_TO_TYPE[tc]]]
|
|
|
|
|
|
# ── metrics ──────────────────────────────────────────────────────────────────
|
|
|
|
def _tol(gt):
|
|
if gt["quant"] and gt["quant"] != "none":
|
|
levels = S.QUANT_LEVELS[gt["quant"]]
|
|
rng = gt["range_max"] - gt["range_min"]
|
|
step = rng / levels
|
|
# One full quantisation level: the correctness bound for lossy quant when
|
|
# the encode rounding convention (round vs truncate) is unknown. Gross
|
|
# corruption is many levels off; a faithful round-trip is ≤1 level.
|
|
return step + 1e-6 * abs(rng), step
|
|
if gt["type"] in S.FLOAT_TYPES:
|
|
return 1e-3, 0.0 # float round-trip epsilon
|
|
return 0.5, 0.0 # integer: rounding-exact (within 0.5)
|
|
|
|
|
|
def nearest_err(recv_v, truth_v):
|
|
"""Max distance from each received value to the nearest truth value."""
|
|
sv = np.unique(np.sort(truth_v.astype(np.float64)))
|
|
idx = np.searchsorted(sv, recv_v)
|
|
idx = np.clip(idx, 1, len(sv) - 1) if len(sv) > 1 else np.zeros_like(idx)
|
|
lo = sv[np.clip(idx - 1, 0, len(sv) - 1)]
|
|
hi = sv[np.clip(idx, 0, len(sv) - 1)]
|
|
d = np.minimum(np.abs(recv_v - lo), np.abs(recv_v - hi))
|
|
return float(np.max(d)) if d.size else 0.0
|
|
|
|
|
|
def gap_check(t_recv, outlier_mult=10.0, max_outlier_frac=0.05):
|
|
"""Detect large discontiguous holes in a received time series.
|
|
|
|
A handful of gaps a few times the median spacing are normal (bursty
|
|
per-tick live pushes, decimation, LTTB). Returns (ok, gap_frac, n_gaps,
|
|
max_gap): ``gap_frac`` is the fraction of the total capture span consumed
|
|
by gaps larger than ``outlier_mult`` times the median inter-sample gap;
|
|
when that adds up to more than ``max_outlier_frac`` of the whole capture,
|
|
the stream stalled/dropped a chunk rather than merely being decimated.
|
|
"""
|
|
if t_recv.size < 10:
|
|
return True, 0.0, 0, 0.0
|
|
t = np.sort(t_recv.astype(np.float64))
|
|
dt = np.diff(t)
|
|
span = float(t[-1] - t[0])
|
|
med = float(np.median(dt))
|
|
if span <= 0.0 or med <= 0.0:
|
|
return True, 0.0, 0, 0.0
|
|
outliers = dt[dt > outlier_mult * med]
|
|
gap_frac = float(outliers.sum() / span)
|
|
return gap_frac <= max_outlier_frac, gap_frac, int(outliers.size), float(dt.max())
|
|
|
|
|
|
def sine_shape(t, v, freq):
|
|
"""Return (corr, nrmse, amp_fit) for a sinusoid fit at ``freq``."""
|
|
w = 2.0 * np.pi * freq
|
|
A = np.column_stack([np.sin(w * t), np.cos(w * t), np.ones_like(t)])
|
|
coef, *_ = np.linalg.lstsq(A, v, rcond=None)
|
|
fit = A @ coef
|
|
span = float(np.max(v) - np.min(v)) or 1.0
|
|
nrmse = float(np.sqrt(np.mean((v - fit) ** 2)) / span)
|
|
corr = float(np.corrcoef(v, fit)[0, 1]) if np.std(v) > 0 else 1.0
|
|
amp = float(np.hypot(coef[0], coef[1]))
|
|
return corr, nrmse, amp
|
|
|
|
|
|
def best_sine_shape(t, v, freq_nominal, band=0.05, n=41):
|
|
"""Refine ``freq_nominal`` within +/-``band`` (fractional) before fitting.
|
|
|
|
The gross-sanity gate assumes the nominal configured frequency, but
|
|
MARTe2's LinuxTimer RT loop runs at a small, systematic offset from true
|
|
wall-clock time (a few percent at most), which accumulates into visible
|
|
phase drift over a multi-second capture even though every sample value is
|
|
bit-correct. A coarse search for the actual best-fit frequency near the
|
|
nominal value absorbs that clock-rate skew while still rejecting a
|
|
genuinely wrong-frequency or corrupted signal, which collapses correlation
|
|
regardless of the search window. Returns (corr, nrmse, amp, freq_used).
|
|
"""
|
|
candidates = np.linspace(freq_nominal * (1.0 - band),
|
|
freq_nominal * (1.0 + band), n)
|
|
best = None
|
|
for f in candidates:
|
|
corr, nrmse, amp = sine_shape(t, v, f)
|
|
if best is None or corr > best[0]:
|
|
best = (corr, nrmse, amp, float(f))
|
|
return best
|
|
|
|
|
|
def compare_signal(gt, t_recv, v_recv, tap_v=None):
|
|
tol, step = _tol(gt)
|
|
truth_v = gt["v"].astype(np.float64)
|
|
m = {
|
|
"type": gt["type"], "quant": gt["quant"], "formula": gt["formula"],
|
|
"n_truth": int(truth_v.size), "n_recv": int(v_recv.size),
|
|
"quant_step": step, "tol": tol,
|
|
}
|
|
if v_recv.size == 0:
|
|
m["pass"] = False
|
|
m["reason"] = "no received samples"
|
|
return m
|
|
|
|
max_err = nearest_err(v_recv, truth_v)
|
|
m["max_abs_err"] = max_err
|
|
fidelity_ok = max_err <= tol
|
|
m["fidelity_ok"] = bool(fidelity_ok)
|
|
|
|
gap_ok, gap_frac, n_gaps, max_gap = gap_check(t_recv)
|
|
m.update(gap_ok=bool(gap_ok), gap_frac=round(gap_frac, 4),
|
|
n_gaps=n_gaps, max_gap=max_gap)
|
|
if not gap_ok:
|
|
m["reason"] = (f"data hole: {gap_frac:.1%} of capture span in "
|
|
f"{n_gaps} gaps >10x median spacing (max={max_gap:.4g}s)")
|
|
|
|
shape_ok = True
|
|
if gt["formula"] == "sine" and v_recv.size >= 8 and gt["freq"]:
|
|
# Refine the fit frequency within +/-5% of nominal before scoring:
|
|
# MARTe2's LinuxTimer RT loop runs at a small, systematic offset from
|
|
# true wall-clock time (a few percent), which accumulates into
|
|
# visible phase drift over a multi-second capture even though every
|
|
# sample value is bit-correct (see best_sine_shape docstring). This
|
|
# keeps the gate a *gross frequency-sanity* check — a wrong-frequency
|
|
# or corrupted signal still collapses corr regardless of the search
|
|
# window — while absorbing legitimate clock-rate skew.
|
|
corr, nrmse, amp, freq_fit = best_sine_shape(t_recv, v_recv, gt["freq"])
|
|
# Shape is a *gross frequency-sanity gate* plus a *tracked quality
|
|
# metric*, not a tight correctness gate. Signal values are bit-faithful
|
|
# (the fidelity oracle proves that); the gap from a perfect fit is
|
|
# almost entirely x-axis timestamp jitter: the hub assigns wall-clock
|
|
# times without per-sample calibration (Phase-A) and the FULL_ARRAY
|
|
# packed-timestamp decode is incomplete (Phase-A4) — both pending. For a
|
|
# correct sinusoid that yields corr ~0.82-0.98 (more for arrays); a
|
|
# wrong-frequency or corrupted signal collapses to corr ~0.00. So the
|
|
# gate (corr>=0.5, nRMSE<=0.30) reliably rejects gross corruption with a
|
|
# wide margin, while corr/nRMSE are recorded so the report can trend
|
|
# them toward 1.0/0.0 as the timestamping work lands (progression).
|
|
nrmse_tol = 0.30 + (step / (gt["range_max"] - gt["range_min"])
|
|
if gt["quant"] != "none" else 0.0)
|
|
shape_ok = corr >= 0.5 and nrmse <= nrmse_tol
|
|
m.update(corr=corr, nrmse=nrmse, amp_fit=amp, freq_fit=freq_fit,
|
|
nrmse_tol=nrmse_tol, shape_ok=bool(shape_ok),
|
|
shape_gate="gross")
|
|
|
|
fed_ok = True
|
|
if tap_v is not None and tap_v.size:
|
|
fed_err = nearest_err(v_recv, tap_v.astype(np.float64))
|
|
fed_ok = fed_err <= tol
|
|
m.update(fed_err=fed_err, fed_ok=bool(fed_ok))
|
|
|
|
m["pass"] = bool(fidelity_ok and shape_ok and fed_ok and gap_ok)
|
|
return m
|
|
|
|
|
|
# ── driver ────────────────────────────────────────────────────────────────────
|
|
|
|
def validate(scenario, received, tap, checks):
|
|
gt = G.build_ground_truth(scenario)
|
|
recv = read_received(received)
|
|
tap_cols = read_marte_binary(tap) if tap and os.path.exists(tap) else None
|
|
|
|
sigs = {}
|
|
overall = True
|
|
for key, (t, v) in sorted(recv.items()):
|
|
# key is "src:sig" or "src:sig[i]" (per-element array push)
|
|
base = key.split("[")[0]
|
|
if base not in gt:
|
|
sigs[key] = {"pass": True, "note": "no ground truth (skipped)",
|
|
"n_recv": int(v.size)}
|
|
continue
|
|
tap_v = None
|
|
if tap_cols is not None:
|
|
sig_name = base.split(":", 1)[1]
|
|
if sig_name in tap_cols:
|
|
tap_v = tap_cols[sig_name].reshape(-1)
|
|
m = compare_signal(gt[base], t, v, tap_v)
|
|
sigs[key] = m
|
|
overall = overall and m.get("pass", False)
|
|
|
|
# roll in client checks
|
|
client = {}
|
|
if checks and os.path.exists(checks):
|
|
with open(checks) as f:
|
|
client = json.load(f)
|
|
live_ok = client.get("live", {}).get("ok", False)
|
|
zoom_ok = all(z.get("inrange", False) for z in client.get("zoom", [])) \
|
|
if client.get("zoom") else True
|
|
win = client.get("window", {})
|
|
window_ok = win.get("ok", True) if win.get("returned", 0) else True
|
|
trig = client.get("trigger", [])
|
|
trig_ok = all(t.get("fired", False) and t.get("windowOk", False)
|
|
for t in trig) if trig else True
|
|
overall = overall and live_ok and zoom_ok and window_ok and trig_ok
|
|
client["_rollup"] = {"live_ok": live_ok, "zoom_ok": zoom_ok,
|
|
"window_ok": window_ok, "trigger_ok": trig_ok}
|
|
|
|
return {"scenario": scenario["id"], "signals": sigs,
|
|
"client": client, "pass": bool(overall)}
|
|
|
|
|
|
def _selftest():
|
|
import math
|
|
t = np.linspace(0, 1, 200)
|
|
truth = np.sin(2 * np.pi * 3 * t)
|
|
gt = {"v": truth, "type": "float32", "quant": "uint16", "formula": "sine",
|
|
"freq": 3.0, "range_min": -1.0, "range_max": 1.0, "elements": 1,
|
|
"rows": 200, "is_time": False, "t": t, "dt": t[1] - t[0]}
|
|
_, step = _tol(gt)
|
|
good = truth + (np.random.rand(200) - 0.5) * step # ≤ step/2
|
|
bad = truth + (np.random.rand(200) - 0.5) * step * 8 # ≫ step
|
|
mg = compare_signal(gt, t, good)
|
|
mb = compare_signal(gt, t, bad)
|
|
assert mg["pass"], f"good should pass: {mg}"
|
|
assert not mb["pass"], f"bad should fail: {mb}"
|
|
# bit-exact integer
|
|
gi = dict(gt); gi.update(type="uint32", quant="none",
|
|
v=np.arange(200, dtype=np.float64), formula="counter")
|
|
mi = compare_signal(gi, t, np.arange(50, 150, dtype=np.float64))
|
|
assert mi["pass"], f"int subset should pass: {mi}"
|
|
mi2 = compare_signal(gi, t, np.array([1.5, 250.0]))
|
|
assert not mi2["pass"], f"int off-grid should fail: {mi2}"
|
|
print("selftest OK")
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description="Validate received waveform")
|
|
p.add_argument("--selftest", action="store_true")
|
|
p.add_argument("--scenario")
|
|
p.add_argument("--received")
|
|
p.add_argument("--tap", default=None)
|
|
p.add_argument("--checks", default=None)
|
|
p.add_argument("--out", default=None)
|
|
args = p.parse_args()
|
|
|
|
if args.selftest:
|
|
_selftest()
|
|
sys.exit(0)
|
|
|
|
sc = next((s for s in S.SCENARIOS if s["id"] == args.scenario), None)
|
|
if sc is None:
|
|
print(f"unknown scenario {args.scenario}", file=sys.stderr)
|
|
sys.exit(2)
|
|
res = validate(sc, args.received, args.tap, args.checks)
|
|
if args.out:
|
|
with open(args.out, "w") as f:
|
|
json.dump(res, f, indent=2)
|
|
print(f"{sc['id']}: {'PASS' if res['pass'] else 'FAIL'}")
|
|
for k, m in res["signals"].items():
|
|
print(f" {k}: pass={m.get('pass')} "
|
|
f"err={m.get('max_abs_err','-')} corr={m.get('corr','-')}")
|
|
sys.exit(0 if res["pass"] else 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|