refactor(e2e): rename Test/E2E/chain -> Test/E2E/suite, run_chain_e2e.sh -> run_e2e.sh
This commit is contained in:
Martino Ferrari
@@ -0,0 +1,332 @@
|
||||
// E2E_Report.typ — Streaming-chain E2E report.
|
||||
//
|
||||
// Renders report_data.json (produced by report_build.py) into a single PDF:
|
||||
// headline KPIs, per-field progression/regression vs the previous run, unit-test
|
||||
// suites, code coverage, performance (CPU/peak-RSS/throughput), per-scenario
|
||||
// waveform fidelity, trend plots, and embedded per-scenario waveform images.
|
||||
//
|
||||
// Compile from the directory holding report_data.json + the *.png artifacts:
|
||||
// typst compile E2E_Report.typ E2E_Report.pdf
|
||||
// Missing inputs degrade to "n/a" / skipped sections so a partial run still
|
||||
// renders.
|
||||
|
||||
#let data = json("report_data.json")
|
||||
#let meta = data.meta
|
||||
#let e2e = data.e2e
|
||||
#let ut = data.unit_tests
|
||||
#let cov = data.coverage
|
||||
#let reg = data.regression
|
||||
#let hl = data.headline
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────
|
||||
#let fmt(v, suffix: "") = if v == none { "n/a" } else { str(v) + suffix }
|
||||
#let fnum(v, digits: 3, suffix: "") = {
|
||||
if v == none { "n/a" } else {
|
||||
[#calc.round(v, digits: digits)] + suffix
|
||||
}
|
||||
}
|
||||
#let ok_color = rgb("#1a7f37")
|
||||
#let bad_color = rgb("#cf222e")
|
||||
#let neutral = rgb("#57606a")
|
||||
|
||||
#let warn_color = rgb("#9a6700") // XFAIL — expected/known failure
|
||||
#let xpass_color = rgb("#8250df") // XPASS — stale marker, needs attention
|
||||
#let status_badge(s) = {
|
||||
let c = if s == "PASS" { ok_color }
|
||||
else if s == "FAIL" { bad_color }
|
||||
else if s == "XFAIL" { warn_color }
|
||||
else if s == "XPASS" { xpass_color }
|
||||
else { neutral }
|
||||
box(fill: c, inset: (x: 6pt, y: 2pt), radius: 3pt, text(fill: white, weight: "bold", size: 8pt)[#s])
|
||||
}
|
||||
|
||||
#let yesno(b) = {
|
||||
if b == true { text(fill: ok_color)[✓] }
|
||||
else if b == false { text(fill: bad_color)[✗] }
|
||||
else { text(fill: neutral)[—] }
|
||||
}
|
||||
|
||||
// progression arrow for a regression row
|
||||
#let trend_arrow(r) = {
|
||||
if r.delta == none or r.delta == 0 { text(fill: neutral)[—] }
|
||||
else {
|
||||
let up = r.delta > 0
|
||||
let good = r.better == true
|
||||
let c = if good { ok_color } else { bad_color }
|
||||
let arrow = if up { "▲" } else { "▼" }
|
||||
text(fill: c)[#arrow #calc.round(r.delta, digits: 4)]
|
||||
}
|
||||
}
|
||||
|
||||
// ── page setup ───────────────────────────────────────────────────────────────
|
||||
#set page(
|
||||
paper: "a4", margin: (x: 1.8cm, y: 1.8cm),
|
||||
header: align(right, text(size: 8pt, fill: neutral)[Streaming-chain E2E · #meta.git_sha]),
|
||||
footer: context align(center, text(size: 8pt, fill: neutral)[#counter(page).display("1 / 1", both: true)]),
|
||||
)
|
||||
#set text(font: "DejaVu Sans", size: 9.5pt)
|
||||
#set heading(numbering: "1.1")
|
||||
#show heading.where(level: 1): it => block(above: 14pt, below: 8pt, text(size: 14pt, weight: "bold", it))
|
||||
#show heading.where(level: 2): it => block(above: 10pt, below: 6pt, text(size: 11pt, weight: "bold", it))
|
||||
|
||||
// ── title ────────────────────────────────────────────────────────────────────
|
||||
#align(center)[
|
||||
#text(size: 20pt, weight: "bold")[MARTe2 Streaming-Chain E2E Report] \
|
||||
#v(2pt)
|
||||
#text(size: 10pt, fill: neutral)[
|
||||
#meta.timestamp · commit #raw(meta.git_sha) · target #meta.target ·
|
||||
run \##data.history_len
|
||||
]
|
||||
]
|
||||
|
||||
#v(6pt)
|
||||
#align(center, status_badge(e2e.overall) + h(8pt) + text(size: 11pt)[
|
||||
E2E #hl.e2e_pass/#hl.e2e_total · Units #hl.unit_pass/#hl.unit_total
|
||||
#if hl.at("e2e_xfail", default: 0) > 0 [ · #hl.e2e_xfail xfail]
|
||||
#if hl.at("e2e_xpass", default: 0) > 0 [ · #text(fill: xpass_color)[#hl.e2e_xpass xpass!]]
|
||||
])
|
||||
|
||||
// ── headline KPIs ────────────────────────────────────────────────────────────
|
||||
#v(10pt)
|
||||
#let kpi(label, value) = box(width: 100%, inset: 8pt, radius: 4pt, fill: rgb("#f6f8fa"))[
|
||||
#align(center)[
|
||||
#text(size: 14pt, weight: "bold")[#value] \
|
||||
#text(size: 8pt, fill: neutral)[#label]
|
||||
]
|
||||
]
|
||||
#grid(columns: 4, gutter: 6pt,
|
||||
kpi("E2E passed", [#hl.e2e_pass/#hl.e2e_total]),
|
||||
kpi("Unit passed", [#hl.unit_pass/#hl.unit_total]),
|
||||
kpi("Mean sine corr", fnum(hl.mean_corr, digits: 4)),
|
||||
kpi("Throughput", fnum(hl.mean_throughput_sps, digits: 0, suffix: " sp/s")),
|
||||
kpi("Python cov", fmt(hl.cov_python, suffix: "%")),
|
||||
kpi("Go cov", fmt(hl.cov_go, suffix: "%")),
|
||||
kpi("Mean peak RSS", fnum(hl.mean_peak_rss_mb, digits: 1, suffix: " MB")),
|
||||
kpi("Mean CPU", fnum(hl.mean_cpu_s, digits: 2, suffix: " s")),
|
||||
)
|
||||
|
||||
// ── progression / regression ─────────────────────────────────────────────────
|
||||
= Progression / regression
|
||||
#if data.is_first_run [
|
||||
_First recorded run — baseline established; no previous run to compare against._
|
||||
] else [
|
||||
Comparison against the previous run (#text(fill: ok_color)[▲] better,
|
||||
#text(fill: bad_color)[▼] worse, — unchanged/unavailable).
|
||||
#v(4pt)
|
||||
#table(
|
||||
columns: (1.6fr, 1fr, 1fr, 1.2fr),
|
||||
align: (left, right, right, right),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 5pt,
|
||||
table.header([*Metric*], [*Current*], [*Previous*], [*Δ (trend)*]),
|
||||
..reg.map(r => (
|
||||
[#r.name],
|
||||
fnum(r.current, digits: 4),
|
||||
fnum(r.previous, digits: 4),
|
||||
trend_arrow(r),
|
||||
)).flatten()
|
||||
)
|
||||
]
|
||||
|
||||
// ── unit tests ───────────────────────────────────────────────────────────────
|
||||
= Unit tests
|
||||
#table(
|
||||
columns: (2fr, 0.8fr, 0.8fr, 0.8fr, 0.8fr, 0.8fr, 0.8fr),
|
||||
align: (left, center, right, right, right, right, right),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 5pt,
|
||||
table.header([*Suite*], [*Status*], [*Total*], [*Pass*], [*Fail*], [*Skip*], [*Time (s)*]),
|
||||
..ut.suites.map(s => (
|
||||
[#s.name],
|
||||
if not s.avail { status_badge("SKIP") } else if s.ok { status_badge("PASS") } else { status_badge("FAIL") },
|
||||
[#s.total], [#s.passed], [#s.failed], [#s.skipped], fnum(s.time_s, digits: 2),
|
||||
)).flatten(),
|
||||
table.cell(colspan: 2)[*Totals*],
|
||||
[#ut.totals.total], [#ut.totals.passed], [#ut.totals.failed], [#ut.totals.skipped], [],
|
||||
)
|
||||
|
||||
// ── coverage ─────────────────────────────────────────────────────────────────
|
||||
= Code coverage
|
||||
#table(
|
||||
columns: (1fr, 1fr, 2fr),
|
||||
align: (left, right, left),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 5pt,
|
||||
table.header([*Language*], [*Coverage*], [*Source*]),
|
||||
..cov.languages.map(l => (
|
||||
[#l.name],
|
||||
if l.pct == none { text(fill: neutral)[n/a] } else { [#l.pct%] },
|
||||
text(size: 8pt, fill: neutral)[#l.note],
|
||||
)).flatten()
|
||||
)
|
||||
|
||||
// per-file C++ detail — the streaming chain's most critical code, so the report
|
||||
// shows line coverage for every project C++ source (worst-covered first).
|
||||
#let cpp = cov.languages.find(l => l.name == "C++")
|
||||
#if cpp != none and cpp.at("files", default: ()).len() > 0 [
|
||||
== C++ source detail
|
||||
#let cf = cpp.files
|
||||
#text(size: 8pt, fill: neutral)[
|
||||
#cpp.at("lines_hit", default: 0) of #cpp.at("lines_found", default: 0)
|
||||
lines covered across #cf.len() project files (worst-covered first;
|
||||
green ≥75%, amber 50–75%, red under 50%).
|
||||
]
|
||||
#v(4pt)
|
||||
#let covcell(p) = {
|
||||
let c = if p == none { neutral }
|
||||
else if p >= 75 { ok_color }
|
||||
else if p >= 50 { warn_color }
|
||||
else { bad_color }
|
||||
text(fill: c, weight: "bold")[#if p == none { "n/a" } else [#p%]]
|
||||
}
|
||||
#table(
|
||||
columns: (3.2fr, 1fr, 1fr, 1.1fr),
|
||||
align: (left, right, right, right),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 4pt,
|
||||
table.header([*Source file*], [*Lines*], [*Hit*], [*Coverage*]),
|
||||
..cf.map(f => (
|
||||
text(size: 8pt)[#f.path],
|
||||
[#f.lines_found],
|
||||
[#f.lines_hit],
|
||||
covcell(f.pct),
|
||||
)).flatten()
|
||||
)
|
||||
]
|
||||
|
||||
// ── performance ──────────────────────────────────────────────────────────────
|
||||
= Performance
|
||||
Per-scenario CPU time and peak resident memory (VmHWM) of the StreamHub and
|
||||
MARTe2 processes, plus sustained client throughput (recorded samples ÷ duration).
|
||||
#v(4pt)
|
||||
#table(
|
||||
columns: (1.8fr, 1fr, 1fr, 1fr, 1fr, 1.1fr),
|
||||
align: (left, right, right, right, right, right),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 5pt,
|
||||
table.header([*Scenario*], [*Hub CPU (s)*], [*Hub RSS (MB)*],
|
||||
[*MARTe CPU (s)*], [*MARTe RSS (MB)*], [*Throughput (sp/s)*]),
|
||||
..e2e.scenarios.map(sc => {
|
||||
let h = sc.perf.at("hub", default: (:))
|
||||
let m = sc.perf.at("marte", default: (:))
|
||||
(
|
||||
[#sc.id],
|
||||
fnum(h.at("cpu_s", default: none), digits: 2),
|
||||
fnum(h.at("peak_rss_mb", default: none), digits: 1),
|
||||
fnum(m.at("cpu_s", default: none), digits: 2),
|
||||
fnum(m.at("peak_rss_mb", default: none), digits: 1),
|
||||
fnum(sc.throughput_sps, digits: 0),
|
||||
)
|
||||
}).flatten()
|
||||
)
|
||||
|
||||
// ── per-scenario waveform fidelity ───────────────────────────────────────────
|
||||
= Scenarios
|
||||
#for sc in e2e.scenarios [
|
||||
== #sc.id #h(6pt) #status_badge(sc.status)
|
||||
#if sc.at("desc", default: none) != none [
|
||||
#v(1pt)
|
||||
#text(size: 9pt, fill: neutral, style: "italic")[#sc.desc]
|
||||
#v(2pt)
|
||||
]
|
||||
#if sc.at("known_issue", default: none) != none [
|
||||
#v(2pt)
|
||||
#box(fill: rgb("#fff8c5"), inset: 5pt, radius: 3pt, width: 100%,
|
||||
text(size: 8pt, fill: rgb("#7d4e00"))[*Known issue:* #sc.known_issue])
|
||||
]
|
||||
#let rb = sc.rollup
|
||||
Client checks:
|
||||
live #yesno(rb.at("live_ok", default: none)) ·
|
||||
zoom #yesno(rb.at("zoom_ok", default: none)) ·
|
||||
window #yesno(rb.at("window_ok", default: none)) ·
|
||||
trigger #yesno(rb.at("trigger_ok", default: none))
|
||||
#if sc.live_frames != none [ · #sc.live_frames live frames]
|
||||
#v(3pt)
|
||||
#table(
|
||||
columns: (1.6fr, 0.6fr, 0.8fr, 0.8fr, 1fr, 0.9fr, 0.9fr, 0.8fr, 0.8fr),
|
||||
align: (left, center, left, left, right, right, right, center, center),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 4pt,
|
||||
table.header([*Signal*], [*Pass*], [*Type*], [*Quant*], [*Max abs err*],
|
||||
[*Corr*], [*nRMSE*], [*Fidelity*], [*Shape*]),
|
||||
..sc.signals.map(g => (
|
||||
raw(g.key),
|
||||
yesno(g.pass),
|
||||
text(size: 8pt)[#g.type],
|
||||
text(size: 8pt)[#g.quant],
|
||||
fnum(g.max_abs_err, digits: 6),
|
||||
fnum(g.corr, digits: 4),
|
||||
fnum(g.nrmse, digits: 4),
|
||||
yesno(g.fidelity_ok),
|
||||
yesno(g.shape_ok),
|
||||
)).flatten()
|
||||
)
|
||||
// zoom / window / trigger behavioural detail (real WS-driven results)
|
||||
#let zooms = sc.at("zoom", default: ())
|
||||
#let win = sc.at("window", default: (:))
|
||||
#let trigs = sc.at("trigger", default: ())
|
||||
#if zooms.len() > 0 or win.len() > 0 [
|
||||
#v(3pt)
|
||||
#text(size: 8.5pt, weight: "bold")[Zoom / window queries]
|
||||
#table(
|
||||
columns: (1fr, 1.2fr, 1fr, 0.8fr, 0.8fr),
|
||||
align: (left, left, right, right, center),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 4pt,
|
||||
table.header([*Query*], [*Signal*], [*Span (s)*], [*Pts*], [*OK*]),
|
||||
..zooms.enumerate().map(((i, z)) => {
|
||||
let r = z.at("range", default: (0, 0))
|
||||
(
|
||||
[zoom #str(i + 1)],
|
||||
raw(z.at("key", default: "")),
|
||||
fnum(r.at(1) - r.at(0), digits: 4),
|
||||
[#z.at("returned", default: 0)],
|
||||
yesno(z.at("inrange", default: none)),
|
||||
)
|
||||
}).flatten(),
|
||||
..(if win.len() > 0 and win.at("returned", default: 0) > 0 {(
|
||||
[window #fnum(win.at("windowSec", default: 0), digits: 2)],
|
||||
raw(win.at("key", default: "")),
|
||||
fnum(win.at("span", default: 0), digits: 4),
|
||||
[#win.at("returned", default: 0)],
|
||||
yesno(win.at("ok", default: none)),
|
||||
)} else {()}),
|
||||
)
|
||||
]
|
||||
#if trigs.len() > 0 [
|
||||
#v(3pt)
|
||||
#text(size: 8.5pt, weight: "bold")[Trigger captures] #h(4pt)
|
||||
#text(size: 8pt, fill: neutral)[(signal #raw(trigs.at(0).at("key", default: "")))]
|
||||
#table(
|
||||
columns: (1fr, 1fr, 0.8fr, 1fr, 1fr, 0.8fr, 0.8fr),
|
||||
align: (left, left, center, right, right, center, center),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 4pt,
|
||||
table.header([*Edge*], [*Mode*], [*Fired*], [*Pre/Post (s)*],
|
||||
[*Capture pts*], [*Win OK*], [*Edge OK*]),
|
||||
..trigs.map(t => (
|
||||
text(size: 8pt)[#t.at("edge", default: "")],
|
||||
text(size: 8pt)[#t.at("mode", default: "")],
|
||||
yesno(t.at("fired", default: none)),
|
||||
text(size: 8pt)[#fnum(t.at("preSec", default: 0), digits: 3) / #fnum(t.at("postSec", default: 0), digits: 3)],
|
||||
[#t.at("capturePts", default: 0)],
|
||||
yesno(t.at("windowOk", default: none)),
|
||||
yesno(t.at("edgeOk", default: none)),
|
||||
)).flatten()
|
||||
)
|
||||
]
|
||||
// embed the waveform overview if report_build.py found one in the artifacts
|
||||
#if sc.at("wave_img", default: none) != none [
|
||||
#v(3pt)
|
||||
#align(center, image(sc.wave_img, width: 92%))
|
||||
]
|
||||
#v(6pt)
|
||||
]
|
||||
|
||||
// ── trend plots ──────────────────────────────────────────────────────────────
|
||||
#if data.trend_plots.len() > 0 [
|
||||
= Trends over runs
|
||||
#grid(columns: 2, gutter: 8pt,
|
||||
..data.trend_plots.map(p => image(p, width: 100%))
|
||||
)
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Executable
BIN
Binary file not shown.
@@ -0,0 +1,7 @@
|
||||
module chain-client
|
||||
|
||||
go 1.21
|
||||
|
||||
require github.com/gorilla/websocket v1.5.1
|
||||
|
||||
require golang.org/x/net v0.17.0 // indirect
|
||||
@@ -0,0 +1,4 @@
|
||||
github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
|
||||
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
|
||||
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
|
||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
@@ -0,0 +1,871 @@
|
||||
// Command chain-client is the authoritative mock StreamHub client for the
|
||||
// streaming-chain E2E suite. It connects to a running StreamHub, records the
|
||||
// live binary stream to disk (received_<id>.bin), and runs behavioural checks
|
||||
// (live / zoom / window / trigger), writing the results to checks_<id>.json.
|
||||
//
|
||||
// Unlike the streamhub smoke test, individual check failures are *recorded*
|
||||
// (not fatal): only connection/protocol corruption exits non-zero, so one
|
||||
// scenario's quirk never aborts the matrix. The waveform validator and report
|
||||
// decide overall pass/fail from checks_<id>.json + the recorded stream.
|
||||
//
|
||||
// received_<id>.bin format ("RCV1"):
|
||||
//
|
||||
// magic[4]="RCV1"; [u32 nSig]; per signal:
|
||||
// [u16 keyLen][key][u32 N][N×f64 t][N×f64 v]
|
||||
//
|
||||
// where each signal's samples are merged across all v1 pushes, sorted by time
|
||||
// and de-duplicated (same timestamp kept once).
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
var (
|
||||
hubFlag = flag.String("hub", "127.0.0.1:8090", "StreamHub host:port")
|
||||
scenario = flag.String("scenario", "", "scenario id (artifact basename)")
|
||||
trigsig = flag.String("trigsig", "", "trigger signal key src:sig (empty = skip)")
|
||||
trigthr = flag.Float64("trigthr", math.NaN(), "trigger threshold (NaN = use mean)")
|
||||
checksCSV = flag.String("checks", "live,zoom,window,trigger", "checks to run")
|
||||
outDir = flag.String("out", "/tmp/chain_e2e", "artifact output dir")
|
||||
durSec = flag.Float64("dur", 4.0, "live recording duration (s)")
|
||||
timeout = flag.Duration("timeout", 90*time.Second, "overall timeout")
|
||||
verbose = flag.Bool("v", false, "log every event")
|
||||
mode = flag.String("mode", "checks", "checks | stress")
|
||||
reqrate = flag.Float64("reqrate", 0, "stress: sustained zoom requests/sec (0 = liveness only)")
|
||||
clientID = flag.Int("clientid", 0, "stress: parallel client index (output suffix)")
|
||||
)
|
||||
|
||||
// ── wire types ────────────────────────────────────────────────────────────
|
||||
|
||||
type sourceInfo struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
Addr string `json:"addr"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
type signalInfo struct {
|
||||
Name string `json:"name"`
|
||||
TypeCode uint32 `json:"typeCode"`
|
||||
NumRows uint32 `json:"numRows"`
|
||||
NumCols uint32 `json:"numCols"`
|
||||
TimeMode int `json:"timeMode"`
|
||||
Rate float64 `json:"samplingRate"`
|
||||
}
|
||||
|
||||
type points struct {
|
||||
T []float64 `json:"t"`
|
||||
V []float64 `json:"v"`
|
||||
}
|
||||
|
||||
type event struct {
|
||||
Type string `json:"type"`
|
||||
Sources json.RawMessage `json:"sources"`
|
||||
SourceID string `json:"sourceId"`
|
||||
Signals json.RawMessage `json:"signals"`
|
||||
ReqID uint32 `json:"reqId"`
|
||||
State string `json:"state"`
|
||||
TrigTime float64 `json:"trigTime"`
|
||||
}
|
||||
|
||||
type pushFrame struct {
|
||||
sourceID string
|
||||
signals map[string]points
|
||||
}
|
||||
|
||||
type captureFrame struct {
|
||||
trigTime, preSec, postSec float64
|
||||
signals map[string]points
|
||||
}
|
||||
|
||||
// ── binary parsers (from Test/E2E/streamhub/main.go) ────────────────────────
|
||||
|
||||
func parsePush(b []byte) (*pushFrame, error) {
|
||||
if len(b) < 2 || b[0] != 1 {
|
||||
return nil, fmt.Errorf("not a v1 frame")
|
||||
}
|
||||
idLen := int(b[1])
|
||||
off := 2
|
||||
if len(b) < off+idLen+4 {
|
||||
return nil, fmt.Errorf("truncated header")
|
||||
}
|
||||
f := &pushFrame{sourceID: string(b[off : off+idLen]), signals: map[string]points{}}
|
||||
off += idLen
|
||||
nSig := int(binary.LittleEndian.Uint32(b[off:]))
|
||||
off += 4
|
||||
for s := 0; s < nSig; s++ {
|
||||
if len(b) < off+2 {
|
||||
return nil, fmt.Errorf("truncated keyLen (sig %d)", s)
|
||||
}
|
||||
keyLen := int(binary.LittleEndian.Uint16(b[off:]))
|
||||
off += 2
|
||||
if len(b) < off+keyLen+4 {
|
||||
return nil, fmt.Errorf("truncated key (sig %d)", s)
|
||||
}
|
||||
key := string(b[off : off+keyLen])
|
||||
off += keyLen
|
||||
n := int(binary.LittleEndian.Uint32(b[off:]))
|
||||
off += 4
|
||||
if len(b) < off+16*n {
|
||||
return nil, fmt.Errorf("truncated data (sig %s n=%d)", key, n)
|
||||
}
|
||||
pts := points{T: make([]float64, n), V: make([]float64, n)}
|
||||
for i := 0; i < n; i++ {
|
||||
pts.T[i] = math.Float64frombits(binary.LittleEndian.Uint64(b[off+8*i:]))
|
||||
}
|
||||
off += 8 * n
|
||||
for i := 0; i < n; i++ {
|
||||
pts.V[i] = math.Float64frombits(binary.LittleEndian.Uint64(b[off+8*i:]))
|
||||
}
|
||||
off += 8 * n
|
||||
f.signals[key] = pts
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func parseCapture(b []byte) (*captureFrame, error) {
|
||||
if len(b) < 1+24+4 || b[0] != 2 {
|
||||
return nil, fmt.Errorf("not a v2 frame")
|
||||
}
|
||||
rd := func(off int) float64 { return math.Float64frombits(binary.LittleEndian.Uint64(b[off:])) }
|
||||
f := &captureFrame{trigTime: rd(1), preSec: rd(9), postSec: rd(17), signals: map[string]points{}}
|
||||
off := 25
|
||||
nSig := int(binary.LittleEndian.Uint32(b[off:]))
|
||||
off += 4
|
||||
for s := 0; s < nSig; s++ {
|
||||
keyLen := int(binary.LittleEndian.Uint16(b[off:]))
|
||||
off += 2
|
||||
key := string(b[off : off+keyLen])
|
||||
off += keyLen
|
||||
n := int(binary.LittleEndian.Uint32(b[off:]))
|
||||
off += 4
|
||||
if len(b) < off+16*n {
|
||||
return nil, fmt.Errorf("truncated capture (sig %s n=%d)", key, n)
|
||||
}
|
||||
pts := points{T: make([]float64, n), V: make([]float64, n)}
|
||||
for i := 0; i < n; i++ {
|
||||
pts.T[i] = math.Float64frombits(binary.LittleEndian.Uint64(b[off+8*i:]))
|
||||
}
|
||||
off += 8 * n
|
||||
for i := 0; i < n; i++ {
|
||||
pts.V[i] = math.Float64frombits(binary.LittleEndian.Uint64(b[off+8*i:]))
|
||||
}
|
||||
off += 8 * n
|
||||
f.signals[key] = pts
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// ── client ──────────────────────────────────────────────────────────────────
|
||||
|
||||
type client struct {
|
||||
ws *websocket.Conn
|
||||
deadline time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
sources []sourceInfo
|
||||
configs map[string][]signalInfo
|
||||
pushes []*pushFrame
|
||||
zooms map[uint32]map[string]points
|
||||
zoomArrival map[uint32]time.Time
|
||||
trigSt []string
|
||||
captures []*captureFrame
|
||||
|
||||
readErr error
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func (c *client) send(v interface{}) {
|
||||
b, _ := json.Marshal(v)
|
||||
if err := c.ws.WriteMessage(websocket.TextMessage, b); err != nil {
|
||||
fatal("ws write: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// reader runs in its own goroutine for the lifetime of the connection. gorilla's
|
||||
// websocket connection cannot survive a read deadline (the next ReadMessage
|
||||
// panics), so we never set one here: we block on ReadMessage and let the overall
|
||||
// timeout/cond logic in waitFor decide when enough has arrived.
|
||||
func (c *client) reader() {
|
||||
defer close(c.done)
|
||||
for {
|
||||
mt, data, err := c.ws.ReadMessage()
|
||||
if err != nil {
|
||||
c.mu.Lock()
|
||||
c.readErr = err
|
||||
c.mu.Unlock()
|
||||
return
|
||||
}
|
||||
c.handle(mt, data)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *client) handle(mt int, data []byte) {
|
||||
switch mt {
|
||||
case websocket.BinaryMessage:
|
||||
if len(data) == 0 {
|
||||
return
|
||||
}
|
||||
switch data[0] {
|
||||
case 1:
|
||||
if f, err := parsePush(data); err == nil {
|
||||
c.mu.Lock()
|
||||
c.pushes = append(c.pushes, f)
|
||||
c.mu.Unlock()
|
||||
} else {
|
||||
fatal("bad v1 frame: %v", err)
|
||||
}
|
||||
case 2:
|
||||
if f, err := parseCapture(data); err == nil {
|
||||
c.mu.Lock()
|
||||
c.captures = append(c.captures, f)
|
||||
c.mu.Unlock()
|
||||
} else {
|
||||
fatal("bad v2 frame: %v", err)
|
||||
}
|
||||
default:
|
||||
fatal("unknown binary frame version %d", data[0])
|
||||
}
|
||||
case websocket.TextMessage:
|
||||
var ev event
|
||||
if err := json.Unmarshal(data, &ev); err != nil {
|
||||
return
|
||||
}
|
||||
if *verbose {
|
||||
log.Printf("event %-12s %.140s", ev.Type, data)
|
||||
}
|
||||
switch ev.Type {
|
||||
case "sources":
|
||||
var s []sourceInfo
|
||||
if json.Unmarshal(ev.Sources, &s) == nil {
|
||||
c.mu.Lock()
|
||||
c.sources = s
|
||||
c.mu.Unlock()
|
||||
}
|
||||
case "config":
|
||||
var s []signalInfo
|
||||
if json.Unmarshal(ev.Signals, &s) == nil {
|
||||
c.mu.Lock()
|
||||
c.configs[ev.SourceID] = s
|
||||
c.mu.Unlock()
|
||||
}
|
||||
case "zoom":
|
||||
var body struct {
|
||||
Signals map[string]points `json:"signals"`
|
||||
}
|
||||
if json.Unmarshal(data, &body) == nil {
|
||||
c.mu.Lock()
|
||||
c.zooms[ev.ReqID] = body.Signals
|
||||
c.zoomArrival[ev.ReqID] = time.Now()
|
||||
c.mu.Unlock()
|
||||
}
|
||||
case "triggerState":
|
||||
c.mu.Lock()
|
||||
c.trigSt = append(c.trigSt, ev.State)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// waitFor polls cond (evaluated under the state lock) until it holds or the
|
||||
// deadline passes. It returns early if the reader goroutine died.
|
||||
func (c *client) waitFor(d time.Duration, cond func() bool) bool {
|
||||
end := time.Now().Add(d)
|
||||
if end.After(c.deadline) {
|
||||
end = c.deadline
|
||||
}
|
||||
for time.Now().Before(end) {
|
||||
c.mu.Lock()
|
||||
ok := cond()
|
||||
err := c.readErr
|
||||
c.mu.Unlock()
|
||||
if ok {
|
||||
return true
|
||||
}
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
select {
|
||||
case <-c.done:
|
||||
c.mu.Lock()
|
||||
ok := cond()
|
||||
c.mu.Unlock()
|
||||
return ok
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// zoom returns the recorded zoom reply for reqID, if present.
|
||||
func (c *client) zoom(reqID uint32) (map[string]points, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
z, ok := c.zooms[reqID]
|
||||
return z, ok
|
||||
}
|
||||
|
||||
// lastCapture returns the most recently recorded trigger capture, if any.
|
||||
func (c *client) lastCapture() *captureFrame {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if len(c.captures) == 0 {
|
||||
return nil
|
||||
}
|
||||
return c.captures[len(c.captures)-1]
|
||||
}
|
||||
|
||||
// nCaptures returns the number of trigger captures recorded so far.
|
||||
func (c *client) nCaptures() int {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return len(c.captures)
|
||||
}
|
||||
|
||||
// nPushes returns the number of live push frames recorded so far.
|
||||
func (c *client) nPushes() int {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return len(c.pushes)
|
||||
}
|
||||
|
||||
func fatal(format string, a ...interface{}) {
|
||||
fmt.Printf("FATAL "+format+"\n", a...)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// merged returns time-sorted, de-duplicated samples per "src:sig" key.
|
||||
func (c *client) merged() map[string]points {
|
||||
c.mu.Lock()
|
||||
pushes := append([]*pushFrame(nil), c.pushes...)
|
||||
c.mu.Unlock()
|
||||
tmp := map[string]points{}
|
||||
for _, f := range pushes {
|
||||
for k, p := range f.signals {
|
||||
full := f.sourceID + ":" + k
|
||||
cur := tmp[full]
|
||||
cur.T = append(cur.T, p.T...)
|
||||
cur.V = append(cur.V, p.V...)
|
||||
tmp[full] = cur
|
||||
}
|
||||
}
|
||||
out := map[string]points{}
|
||||
for k, p := range tmp {
|
||||
idx := make([]int, len(p.T))
|
||||
for i := range idx {
|
||||
idx[i] = i
|
||||
}
|
||||
sort.Slice(idx, func(a, b int) bool { return p.T[idx[a]] < p.T[idx[b]] })
|
||||
var op points
|
||||
last := math.NaN()
|
||||
for _, i := range idx {
|
||||
if !math.IsNaN(last) && p.T[i] == last {
|
||||
continue
|
||||
}
|
||||
op.T = append(op.T, p.T[i])
|
||||
op.V = append(op.V, p.V[i])
|
||||
last = p.T[i]
|
||||
}
|
||||
out[k] = op
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ── received_<id>.bin writer ─────────────────────────────────────────────────
|
||||
|
||||
func writeReceived(path string, m map[string]points) error {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
var buf []byte
|
||||
put32 := func(v uint32) { var t [4]byte; binary.LittleEndian.PutUint32(t[:], v); buf = append(buf, t[:]...) }
|
||||
put16 := func(v uint16) { var t [2]byte; binary.LittleEndian.PutUint16(t[:], v); buf = append(buf, t[:]...) }
|
||||
putf := func(v float64) { var t [8]byte; binary.LittleEndian.PutUint64(t[:], math.Float64bits(v)); buf = append(buf, t[:]...) }
|
||||
buf = append(buf, []byte("RCV1")...)
|
||||
put32(uint32(len(keys)))
|
||||
for _, k := range keys {
|
||||
put16(uint16(len(k)))
|
||||
buf = append(buf, []byte(k)...)
|
||||
p := m[k]
|
||||
put32(uint32(len(p.T)))
|
||||
for _, t := range p.T {
|
||||
putf(t)
|
||||
}
|
||||
for _, v := range p.V {
|
||||
putf(v)
|
||||
}
|
||||
}
|
||||
_, err = f.Write(buf)
|
||||
return err
|
||||
}
|
||||
|
||||
// ── checks structures ────────────────────────────────────────────────────────
|
||||
|
||||
type zoomCheck struct {
|
||||
Range [2]float64 `json:"range"`
|
||||
N int `json:"n"`
|
||||
Returned int `json:"returned"`
|
||||
InRange bool `json:"inrange"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
type windowCheck struct {
|
||||
WindowSec float64 `json:"windowSec"`
|
||||
Span float64 `json:"span"`
|
||||
Returned int `json:"returned"`
|
||||
OK bool `json:"ok"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
type trigCheck struct {
|
||||
Edge string `json:"edge"`
|
||||
Mode string `json:"mode"`
|
||||
Fired bool `json:"fired"`
|
||||
TrigTime float64 `json:"trigTime"`
|
||||
PreSec float64 `json:"preSec"`
|
||||
PostSec float64 `json:"postSec"`
|
||||
CapturePts int `json:"capturePts"`
|
||||
EdgeOK bool `json:"edgeOk"`
|
||||
WindowOK bool `json:"windowOk"`
|
||||
Rearmed bool `json:"rearmed"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
type liveCheck struct {
|
||||
OK bool `json:"ok"`
|
||||
Frames int `json:"frames"`
|
||||
Signals int `json:"signals"`
|
||||
Monotonic bool `json:"monotonic"`
|
||||
WallClock bool `json:"wallclock"`
|
||||
DurationS float64 `json:"duration"`
|
||||
}
|
||||
|
||||
type checksOut struct {
|
||||
Scenario string `json:"scenario"`
|
||||
Live liveCheck `json:"live"`
|
||||
Zoom []zoomCheck `json:"zoom"`
|
||||
Window windowCheck `json:"window"`
|
||||
Trigger []trigCheck `json:"trigger"`
|
||||
}
|
||||
|
||||
func has(set, name string) bool {
|
||||
for _, s := range strings.Split(set, ",") {
|
||||
if strings.TrimSpace(s) == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
if *scenario == "" {
|
||||
fatal("missing -scenario")
|
||||
}
|
||||
if err := os.MkdirAll(*outDir, 0o755); err != nil {
|
||||
fatal("mkdir %s: %v", *outDir, err)
|
||||
}
|
||||
url := "ws://" + *hubFlag + "/ws"
|
||||
log.Printf("connecting to %s (scenario %s)", url, *scenario)
|
||||
ws, _, err := websocket.DefaultDialer.Dial(url, nil)
|
||||
if err != nil {
|
||||
fatal("dial %s: %v", url, err)
|
||||
}
|
||||
defer ws.Close()
|
||||
|
||||
c := &client{
|
||||
ws: ws, deadline: time.Now().Add(*timeout),
|
||||
configs: map[string][]signalInfo{},
|
||||
zooms: map[uint32]map[string]points{},
|
||||
zoomArrival: map[uint32]time.Time{},
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
go c.reader()
|
||||
out := checksOut{Scenario: *scenario}
|
||||
|
||||
// 1. sources connected
|
||||
c.send(map[string]interface{}{"type": "getSources"})
|
||||
if !c.waitFor(20*time.Second, func() bool {
|
||||
for _, s := range c.sources {
|
||||
if s.State == "connected" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}) {
|
||||
fatal("no connected source within timeout")
|
||||
}
|
||||
for _, s := range c.sources {
|
||||
c.send(map[string]interface{}{"type": "getConfig", "sourceId": s.ID})
|
||||
}
|
||||
c.waitFor(5*time.Second, func() bool {
|
||||
for _, s := range c.sources {
|
||||
if s.State == "connected" && len(c.configs[s.ID]) == 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
// stress mode: record liveness + sustained zoom latency, then exit. The
|
||||
// correctness checks below are skipped (a separate gate framework).
|
||||
if *mode == "stress" {
|
||||
c.runStress(*scenario, *clientID, *durSec, *reqrate, *outDir)
|
||||
return
|
||||
}
|
||||
|
||||
// 2. live recording — the reader goroutine accumulates pushes in the
|
||||
// background, so we simply wait out the recording window (or an early
|
||||
// reader death).
|
||||
c.waitFor(time.Duration(*durSec*float64(time.Second)), func() bool { return false })
|
||||
m := c.merged()
|
||||
recvPath := filepath.Join(*outDir, "received_"+*scenario+".bin")
|
||||
if err := writeReceived(recvPath, m); err != nil {
|
||||
fatal("write received: %v", err)
|
||||
}
|
||||
now := float64(time.Now().UnixNano()) / 1e9
|
||||
mono, wall := true, true
|
||||
for _, p := range m {
|
||||
for i, t := range p.T {
|
||||
if math.Abs(t-now) > 60.0 {
|
||||
wall = false
|
||||
}
|
||||
if i > 0 && t < p.T[i-1]-1e-9 {
|
||||
mono = false
|
||||
}
|
||||
}
|
||||
}
|
||||
nPush := c.nPushes()
|
||||
out.Live = liveCheck{
|
||||
OK: nPush >= 5 && len(m) > 0 && mono && wall,
|
||||
Frames: nPush, Signals: len(m), Monotonic: mono,
|
||||
WallClock: wall, DurationS: *durSec,
|
||||
}
|
||||
log.Printf("live: %d frames, %d signals, mono=%v wall=%v", nPush, len(m), mono, wall)
|
||||
|
||||
// busiest signal for zoom/window
|
||||
var busy string
|
||||
bn := 0
|
||||
for k, p := range m {
|
||||
if len(p.T) > bn {
|
||||
bn, busy = len(p.T), k
|
||||
}
|
||||
}
|
||||
|
||||
// 3. zoom: narrow + wide
|
||||
if has(*checksCSV, "zoom") && busy != "" {
|
||||
ts := m[busy].T
|
||||
t1 := ts[len(ts)-1]
|
||||
t0full := ts[0]
|
||||
ranges := [][2]float64{{t1 - 0.05, t1}, {t0full, t1}} // narrow, wide
|
||||
for ri, rg := range ranges {
|
||||
reqID := uint32(1000 + ri)
|
||||
c.send(map[string]interface{}{
|
||||
"type": "zoom", "reqId": reqID, "t0": rg[0], "t1": rg[1],
|
||||
"n": 300, "signals": busy,
|
||||
})
|
||||
ok := c.waitFor(8*time.Second, func() bool { _, ok := c.zooms[reqID]; return ok })
|
||||
zc := zoomCheck{Range: rg, N: 300, Key: busy, InRange: true}
|
||||
if ok {
|
||||
z, _ := c.zoom(reqID)
|
||||
pts := z[busy]
|
||||
zc.Returned = len(pts.T)
|
||||
for _, t := range pts.T {
|
||||
if t < rg[0]-1e-6 || t > rg[1]+1e-6 {
|
||||
zc.InRange = false
|
||||
}
|
||||
}
|
||||
} else {
|
||||
zc.InRange = false
|
||||
}
|
||||
out.Zoom = append(out.Zoom, zc)
|
||||
log.Printf("zoom [%.4f,%.4f] returned=%d inrange=%v", rg[0], rg[1], zc.Returned, zc.InRange)
|
||||
}
|
||||
}
|
||||
|
||||
// 4. window: zoom over [now-windowSec, now] and check span ≤ window
|
||||
if has(*checksCSV, "window") && busy != "" {
|
||||
ts := m[busy].T
|
||||
t1 := ts[len(ts)-1]
|
||||
const winSec = 1.0
|
||||
reqID := uint32(2000)
|
||||
c.send(map[string]interface{}{
|
||||
"type": "zoom", "reqId": reqID, "t0": t1 - winSec, "t1": t1,
|
||||
"n": 600, "signals": busy,
|
||||
})
|
||||
ok := c.waitFor(8*time.Second, func() bool { _, ok := c.zooms[reqID]; return ok })
|
||||
wc := windowCheck{WindowSec: winSec, Key: busy}
|
||||
if ok {
|
||||
z, _ := c.zoom(reqID)
|
||||
pts := z[busy]
|
||||
wc.Returned = len(pts.T)
|
||||
if len(pts.T) >= 2 {
|
||||
wc.Span = pts.T[len(pts.T)-1] - pts.T[0]
|
||||
wc.OK = wc.Span <= winSec+1e-3
|
||||
}
|
||||
}
|
||||
out.Window = wc
|
||||
log.Printf("window %.2fs: span=%.4f returned=%d ok=%v", winSec, wc.Span, wc.Returned, wc.OK)
|
||||
}
|
||||
|
||||
// 5. trigger matrix
|
||||
if has(*checksCSV, "trigger") && *trigsig != "" {
|
||||
thr := *trigthr
|
||||
if math.IsNaN(thr) {
|
||||
if p, ok := m[*trigsig]; ok && len(p.V) > 0 {
|
||||
s := 0.0
|
||||
for _, v := range p.V {
|
||||
s += v
|
||||
}
|
||||
thr = s / float64(len(p.V))
|
||||
} else {
|
||||
thr = 0.0
|
||||
}
|
||||
}
|
||||
log.Printf("trigger signal %s threshold %.6g", *trigsig, thr)
|
||||
for _, edge := range []string{"rising", "falling", "both"} {
|
||||
for _, mode := range []string{"normal", "single"} {
|
||||
out.Trigger = append(out.Trigger, c.runTrigger(*trigsig, edge, mode, thr))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// write checks json
|
||||
cj := filepath.Join(*outDir, "checks_"+*scenario+".json")
|
||||
b, _ := json.MarshalIndent(out, "", " ")
|
||||
if err := os.WriteFile(cj, b, 0o644); err != nil {
|
||||
fatal("write checks: %v", err)
|
||||
}
|
||||
fmt.Printf("OK chain-client %s: %s + %s\n", *scenario, filepath.Base(recvPath), filepath.Base(cj))
|
||||
}
|
||||
|
||||
// ── stress mode ──────────────────────────────────────────────────────────────
|
||||
|
||||
type stressOut struct {
|
||||
Scenario string `json:"scenario"`
|
||||
ClientID int `json:"clientId"`
|
||||
Frames int `json:"frames"`
|
||||
Signals int `json:"signals"`
|
||||
Monotonic bool `json:"monotonic"`
|
||||
WallClock bool `json:"wallclock"`
|
||||
DurationS float64 `json:"duration"`
|
||||
ReqRate float64 `json:"reqRate"`
|
||||
ZoomCount int `json:"zoomCount"`
|
||||
ZoomFail int `json:"zoomFail"`
|
||||
ZoomP50ms float64 `json:"zoomP50ms"`
|
||||
ZoomP95ms float64 `json:"zoomP95ms"`
|
||||
ZoomMaxms float64 `json:"zoomMaxms"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
// busiestKey returns the full "src:sig" key carrying the most samples so far.
|
||||
func (c *client) busiestKey() string {
|
||||
m := c.merged()
|
||||
var busy string
|
||||
bn := 0
|
||||
for k, p := range m {
|
||||
if len(p.T) > bn {
|
||||
bn, busy = len(p.T), k
|
||||
}
|
||||
}
|
||||
return busy
|
||||
}
|
||||
|
||||
// maxTimeFull returns the latest timestamp seen for a full "src:sig" key.
|
||||
func (c *client) maxTimeFull(full string) (float64, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
var mx float64
|
||||
found := false
|
||||
for _, f := range c.pushes {
|
||||
if !strings.HasPrefix(full, f.sourceID+":") {
|
||||
continue
|
||||
}
|
||||
name := full[len(f.sourceID)+1:]
|
||||
p, ok := f.signals[name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, t := range p.T {
|
||||
if !found || t > mx {
|
||||
mx, found = t, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return mx, found
|
||||
}
|
||||
|
||||
// runStress records liveness for the duration while (optionally) issuing zoom
|
||||
// queries at a sustained rate, measuring round-trip latency. One request is in
|
||||
// flight at a time, so latency reflects the hub's serialised zoom service time
|
||||
// under whatever concurrent live/zoom load the matrix imposes.
|
||||
func (c *client) runStress(scenario string, clientID int, dur, reqrate float64, outDir string) {
|
||||
if !c.waitFor(20*time.Second, func() bool { return len(c.pushes) > 0 }) {
|
||||
fatal("stress: no live push within timeout")
|
||||
}
|
||||
// brief warmup so the busiest signal and a usable time window exist.
|
||||
c.waitFor(500*time.Millisecond, func() bool { return false })
|
||||
key := c.busiestKey()
|
||||
|
||||
start := time.Now()
|
||||
end := start.Add(time.Duration(dur * float64(time.Second)))
|
||||
var lat []float64
|
||||
zoomFail := 0
|
||||
var reqID uint32 = 5000
|
||||
var interval time.Duration
|
||||
if reqrate > 0 {
|
||||
interval = time.Duration(float64(time.Second) / reqrate)
|
||||
}
|
||||
|
||||
for reqrate > 0 && key != "" && time.Now().Before(end) {
|
||||
tick := time.Now()
|
||||
t1, ok := c.maxTimeFull(key)
|
||||
if !ok {
|
||||
c.waitFor(50*time.Millisecond, func() bool { return false })
|
||||
continue
|
||||
}
|
||||
reqID++
|
||||
c.send(map[string]interface{}{
|
||||
"type": "zoom", "reqId": reqID, "t0": t1 - 0.5, "t1": t1,
|
||||
"n": 300, "signals": key,
|
||||
})
|
||||
sendT := time.Now()
|
||||
got := c.waitFor(3*time.Second, func() bool {
|
||||
_, ok := c.zoomArrival[reqID]
|
||||
return ok
|
||||
})
|
||||
if got {
|
||||
c.mu.Lock()
|
||||
at := c.zoomArrival[reqID]
|
||||
c.mu.Unlock()
|
||||
lat = append(lat, at.Sub(sendT).Seconds()*1000.0)
|
||||
} else {
|
||||
zoomFail++
|
||||
}
|
||||
if rem := interval - time.Since(tick); rem > 0 {
|
||||
c.waitFor(rem, func() bool { return false })
|
||||
}
|
||||
}
|
||||
// if no reqrate, simply wait out the remaining liveness window.
|
||||
if reqrate <= 0 {
|
||||
c.waitFor(time.Until(end), func() bool { return false })
|
||||
}
|
||||
|
||||
m := c.merged()
|
||||
now := float64(time.Now().UnixNano()) / 1e9
|
||||
mono, wall := true, true
|
||||
for _, p := range m {
|
||||
for i, t := range p.T {
|
||||
if math.Abs(t-now) > 60.0 {
|
||||
wall = false
|
||||
}
|
||||
if i > 0 && t < p.T[i-1]-1e-9 {
|
||||
mono = false
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Float64s(lat)
|
||||
pct := func(q float64) float64 {
|
||||
if len(lat) == 0 {
|
||||
return 0
|
||||
}
|
||||
idx := int(q * float64(len(lat)-1))
|
||||
return lat[idx]
|
||||
}
|
||||
so := stressOut{
|
||||
Scenario: scenario, ClientID: clientID,
|
||||
Frames: c.nPushes(), Signals: len(m), Monotonic: mono,
|
||||
WallClock: wall, DurationS: dur, ReqRate: reqrate, Key: key,
|
||||
ZoomCount: len(lat), ZoomFail: zoomFail,
|
||||
ZoomP50ms: pct(0.50), ZoomP95ms: pct(0.95),
|
||||
}
|
||||
if len(lat) > 0 {
|
||||
so.ZoomMaxms = lat[len(lat)-1]
|
||||
}
|
||||
path := filepath.Join(outDir,
|
||||
fmt.Sprintf("stress_%s_c%d.json", scenario, clientID))
|
||||
b, _ := json.MarshalIndent(so, "", " ")
|
||||
if err := os.WriteFile(path, b, 0o644); err != nil {
|
||||
fatal("write stress: %v", err)
|
||||
}
|
||||
log.Printf("stress: frames=%d signals=%d zoom n=%d fail=%d p50=%.1fms p95=%.1fms max=%.1fms",
|
||||
so.Frames, so.Signals, so.ZoomCount, so.ZoomFail, so.ZoomP50ms, so.ZoomP95ms, so.ZoomMaxms)
|
||||
fmt.Printf("OK stress %s c%d: %s\n", scenario, clientID, filepath.Base(path))
|
||||
}
|
||||
|
||||
// runTrigger configures one edge/mode trigger, arms it, and records the result.
|
||||
func (c *client) runTrigger(key, edge, mode string, thr float64) trigCheck {
|
||||
tc := trigCheck{Edge: edge, Mode: mode, Key: key}
|
||||
beforeCaps := c.nCaptures()
|
||||
c.send(map[string]interface{}{
|
||||
"type": "setTrigger", "signal": key, "edge": edge,
|
||||
"threshold": thr, "windowSec": 0.1, "prePercent": 20.0, "mode": mode,
|
||||
})
|
||||
c.send(map[string]interface{}{"type": "arm"})
|
||||
fired := c.waitFor(8*time.Second, func() bool { return len(c.captures) > beforeCaps })
|
||||
tc.Fired = fired
|
||||
if !fired {
|
||||
c.send(map[string]interface{}{"type": "disarm"})
|
||||
return tc
|
||||
}
|
||||
cap0 := c.lastCapture()
|
||||
tc.TrigTime, tc.PreSec, tc.PostSec = cap0.trigTime, cap0.preSec, cap0.postSec
|
||||
tc.WindowOK = math.Abs(cap0.preSec-0.02) < 1e-6 && math.Abs(cap0.postSec-0.08) < 1e-6
|
||||
if pts, ok := cap0.signals[key]; ok {
|
||||
tc.CapturePts = len(pts.T)
|
||||
tc.EdgeOK = edgeCrosses(pts, cap0.trigTime, edge, thr)
|
||||
}
|
||||
// re-arm behaviour
|
||||
if mode == "normal" {
|
||||
tc.Rearmed = c.waitFor(4*time.Second, func() bool { return len(c.captures) > beforeCaps+1 })
|
||||
} else { // single: must NOT fire again until rearm
|
||||
again := c.waitFor(1500*time.Millisecond, func() bool { return len(c.captures) > beforeCaps+1 })
|
||||
c.send(map[string]interface{}{"type": "rearm"})
|
||||
tc.Rearmed = !again && c.waitFor(4*time.Second, func() bool { return len(c.captures) > beforeCaps+1 })
|
||||
}
|
||||
c.send(map[string]interface{}{"type": "disarm"})
|
||||
log.Printf("trigger %s/%s fired=%v edgeOk=%v winOk=%v rearm=%v",
|
||||
edge, mode, tc.Fired, tc.EdgeOK, tc.WindowOK, tc.Rearmed)
|
||||
return tc
|
||||
}
|
||||
|
||||
// edgeCrosses verifies the captured waveform crosses thr in the edge direction
|
||||
// near trigTime.
|
||||
func edgeCrosses(p points, trigTime float64, edge string, thr float64) bool {
|
||||
// find the sample pair straddling trigTime
|
||||
for i := 1; i < len(p.T); i++ {
|
||||
if p.T[i-1] <= trigTime && p.T[i] >= trigTime {
|
||||
a, b := p.V[i-1], p.V[i]
|
||||
switch edge {
|
||||
case "rising":
|
||||
return a <= thr && b >= thr
|
||||
case "falling":
|
||||
return a >= thr && b <= thr
|
||||
case "both":
|
||||
return (a <= thr && b >= thr) || (a >= thr && b <= thr)
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ── frame builders (mirror the StreamHub wire format) ───────────────────────
|
||||
|
||||
func putf(b []byte, v float64) []byte {
|
||||
var t [8]byte
|
||||
binary.LittleEndian.PutUint64(t[:], math.Float64bits(v))
|
||||
return append(b, t[:]...)
|
||||
}
|
||||
|
||||
func putSignals(b []byte, sigs map[string]points) []byte {
|
||||
var n [4]byte
|
||||
binary.LittleEndian.PutUint32(n[:], uint32(len(sigs)))
|
||||
b = append(b, n[:]...)
|
||||
for k, p := range sigs {
|
||||
var kl [2]byte
|
||||
binary.LittleEndian.PutUint16(kl[:], uint16(len(k)))
|
||||
b = append(b, kl[:]...)
|
||||
b = append(b, []byte(k)...)
|
||||
var cnt [4]byte
|
||||
binary.LittleEndian.PutUint32(cnt[:], uint32(len(p.T)))
|
||||
b = append(b, cnt[:]...)
|
||||
for _, t := range p.T {
|
||||
b = putf(b, t)
|
||||
}
|
||||
for _, v := range p.V {
|
||||
b = putf(b, v)
|
||||
}
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func buildPush(srcID string, sigs map[string]points) []byte {
|
||||
b := []byte{1, byte(len(srcID))}
|
||||
b = append(b, []byte(srcID)...)
|
||||
return putSignals(b, sigs)
|
||||
}
|
||||
|
||||
func buildCapture(trigTime, preSec, postSec float64, sigs map[string]points) []byte {
|
||||
b := []byte{2}
|
||||
b = putf(b, trigTime)
|
||||
b = putf(b, preSec)
|
||||
b = putf(b, postSec)
|
||||
return putSignals(b, sigs)
|
||||
}
|
||||
|
||||
// ── tests ───────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestParsePushRoundTrip(t *testing.T) {
|
||||
in := map[string]points{"Sine": {T: []float64{0, 0.1, 0.2}, V: []float64{1, 2, 3}}}
|
||||
f, err := parsePush(buildPush("src", in))
|
||||
if err != nil {
|
||||
t.Fatalf("parsePush: %v", err)
|
||||
}
|
||||
if f.sourceID != "src" {
|
||||
t.Errorf("sourceID = %q, want src", f.sourceID)
|
||||
}
|
||||
got := f.signals["Sine"]
|
||||
if len(got.T) != 3 || got.T[2] != 0.2 || got.V[1] != 2 {
|
||||
t.Errorf("bad payload: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePushRejectsWrongVersion(t *testing.T) {
|
||||
if _, err := parsePush([]byte{2, 0, 0}); err == nil {
|
||||
t.Error("expected error on non-v1 frame")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePushTruncated(t *testing.T) {
|
||||
full := buildPush("src", map[string]points{"A": {T: []float64{0}, V: []float64{1}}})
|
||||
if _, err := parsePush(full[:len(full)-4]); err == nil {
|
||||
t.Error("expected truncation error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCaptureRoundTrip(t *testing.T) {
|
||||
in := map[string]points{"Sig": {T: []float64{1, 2}, V: []float64{9, 8}}}
|
||||
f, err := parseCapture(buildCapture(100.5, 0.02, 0.08, in))
|
||||
if err != nil {
|
||||
t.Fatalf("parseCapture: %v", err)
|
||||
}
|
||||
if f.trigTime != 100.5 || f.preSec != 0.02 || f.postSec != 0.08 {
|
||||
t.Errorf("header = %v/%v/%v", f.trigTime, f.preSec, f.postSec)
|
||||
}
|
||||
if g := f.signals["Sig"]; len(g.T) != 2 || g.V[0] != 9 {
|
||||
t.Errorf("bad capture payload: %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCaptureRejectsWrongVersion(t *testing.T) {
|
||||
if _, err := parseCapture([]byte{1, 0, 0, 0}); err == nil {
|
||||
t.Error("expected error on non-v2 frame")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergedSortsAndDedups(t *testing.T) {
|
||||
c := &client{
|
||||
configs: map[string][]signalInfo{},
|
||||
zooms: map[uint32]map[string]points{},
|
||||
}
|
||||
// two pushes, out of order, with one duplicate timestamp
|
||||
c.pushes = []*pushFrame{
|
||||
{sourceID: "s", signals: map[string]points{"A": {T: []float64{0.2, 0.1}, V: []float64{2, 1}}}},
|
||||
{sourceID: "s", signals: map[string]points{"A": {T: []float64{0.1, 0.3}, V: []float64{1, 3}}}},
|
||||
}
|
||||
m := c.merged()
|
||||
got := m["s:A"]
|
||||
want := []float64{0.1, 0.2, 0.3}
|
||||
if len(got.T) != len(want) {
|
||||
t.Fatalf("len = %d, want %d (%+v)", len(got.T), len(want), got)
|
||||
}
|
||||
for i := range want {
|
||||
if got.T[i] != want[i] {
|
||||
t.Errorf("t[%d] = %v, want %v", i, got.T[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEdgeCrosses(t *testing.T) {
|
||||
rising := points{T: []float64{0, 1, 2}, V: []float64{-1, 1, 2}}
|
||||
if !edgeCrosses(rising, 0.5, "rising", 0.0) {
|
||||
t.Error("rising cross not detected")
|
||||
}
|
||||
if edgeCrosses(rising, 0.5, "falling", 0.0) {
|
||||
t.Error("false falling detection on rising data")
|
||||
}
|
||||
falling := points{T: []float64{0, 1}, V: []float64{1, -1}}
|
||||
if !edgeCrosses(falling, 0.5, "falling", 0.0) {
|
||||
t.Error("falling cross not detected")
|
||||
}
|
||||
if !edgeCrosses(falling, 0.5, "both", 0.0) {
|
||||
t.Error("both should match a falling edge")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHas(t *testing.T) {
|
||||
if !has("live,zoom,trigger", "zoom") {
|
||||
t.Error("has should find zoom")
|
||||
}
|
||||
if has("live, window", "trigger") {
|
||||
t.Error("has should not find trigger")
|
||||
}
|
||||
if !has("live, window", "window") {
|
||||
t.Error("has should trim spaces")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,360 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
collect.py — Run the unit-test suites and gather coverage for the E2E report.
|
||||
|
||||
Produces two JSON artifacts in --out:
|
||||
|
||||
* ``unit_tests.json`` — per-suite {total, passed, failed, skipped, time_s, ok}
|
||||
for the C++ GTest binary, the Go chain-client tests, and the Python
|
||||
framework tests, plus grand totals.
|
||||
* ``coverage.json`` — per-language {pct, avail, note} for Python (coverage.py),
|
||||
Go (``go test -cover``) and C++ (lcov, best-effort: only when the build was
|
||||
instrumented with .gcno files; otherwise reported unavailable).
|
||||
|
||||
Each suite is isolated: a missing toolchain or a failing suite is recorded, never
|
||||
fatal, so the report always renders. Requires the MARTe env (LD_LIBRARY_PATH) to
|
||||
already be exported for the GTest binary — the orchestrator does this.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
|
||||
def _run(cmd, cwd=None, env=None, timeout=600):
|
||||
try:
|
||||
p = subprocess.run(cmd, cwd=cwd, env=env, timeout=timeout,
|
||||
capture_output=True, text=True)
|
||||
return p.returncode, p.stdout, p.stderr
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError, OSError) as e:
|
||||
return 127, "", str(e)
|
||||
|
||||
|
||||
# ── C++ GTest ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def gtest_suite(gtest_bin, work):
|
||||
s = {"name": "C++ GTest", "lang": "cpp", "total": 0, "passed": 0,
|
||||
"failed": 0, "skipped": 0, "time_s": 0.0, "ok": False, "avail": False}
|
||||
if not gtest_bin or not os.path.exists(gtest_bin):
|
||||
s["detail"] = "GTest binary not found"
|
||||
return s
|
||||
xml_p = os.path.join(work, "gtest.xml")
|
||||
rc, out, err = _run([gtest_bin, f"--gtest_output=xml:{xml_p}"], timeout=900)
|
||||
s["avail"] = True
|
||||
if os.path.exists(xml_p):
|
||||
try:
|
||||
root = ET.parse(xml_p).getroot()
|
||||
s["total"] = int(root.get("tests", 0))
|
||||
s["failed"] = int(root.get("failures", 0)) + int(root.get("errors", 0))
|
||||
s["skipped"] = int(root.get("disabled", 0)) + int(root.get("skipped", 0))
|
||||
s["time_s"] = float(root.get("time", 0.0))
|
||||
s["passed"] = s["total"] - s["failed"] - s["skipped"]
|
||||
s["ok"] = s["failed"] == 0 and s["total"] > 0
|
||||
except (ET.ParseError, ValueError) as e:
|
||||
s["detail"] = f"xml parse: {e}"
|
||||
else:
|
||||
s["detail"] = (err or out or "no xml produced")[-200:]
|
||||
return s
|
||||
|
||||
|
||||
# ── C++ Integration (DebugService runtime, non-GTest) ─────────────────────────
|
||||
|
||||
def integration_suite(int_bin, work, timeout=220):
|
||||
"""Run the printf-narrated IntegrationTests.ex binary and heuristically
|
||||
derive per-test pass/fail from its stdout.
|
||||
|
||||
This binary predates GTest adoption and always ``return 0`` from main()
|
||||
(only an internal 180s SIGALRM timeout or an OS-level crash produce a
|
||||
non-zero exit), so exit code alone is not a reliable signal. Each of its
|
||||
7 "--- Test N: ..." blocks prints "SUCCESS:"/"VALIDATION SUCCESSFUL:" on
|
||||
success or "ERROR:"/"FAILURE:" on failure, so split stdout by those
|
||||
headers and flag a block failed if it contains an ERROR/FAILURE marker.
|
||||
Exercises DebugServiceBase.cpp/DebugService.cpp runtime logic that the
|
||||
header-only DebugServiceGTest suite deliberately does not touch, so it is
|
||||
the only source of real coverage for those files.
|
||||
"""
|
||||
s = {"name": "C++ Integration", "lang": "cpp", "total": 0, "passed": 0,
|
||||
"failed": 0, "skipped": 0, "time_s": 0.0, "ok": False, "avail": False}
|
||||
if not int_bin or not os.path.exists(int_bin):
|
||||
s["detail"] = "IntegrationTests binary not found"
|
||||
return s
|
||||
s["avail"] = True
|
||||
t0 = time.time()
|
||||
rc, out, err = _run([int_bin], timeout=timeout)
|
||||
s["time_s"] = round(time.time() - t0, 1)
|
||||
text = out + "\n" + err
|
||||
blocks = re.split(r"\n(?=--- Test \d+:)", text)
|
||||
test_blocks = [b for b in blocks if b.lstrip().startswith("--- Test")]
|
||||
finished = "All Integration Tests Finished." in text
|
||||
s["total"] = len(test_blocks)
|
||||
s["failed"] = sum(1 for b in test_blocks if re.search(r"\b(ERROR|FAILURE):", b))
|
||||
if not finished and s["total"] == 0:
|
||||
# Crashed/timed out before printing anything useful.
|
||||
s["total"] = 1
|
||||
s["failed"] = 1
|
||||
s["detail"] = f"binary did not complete (rc={rc}): {(err or out)[-200:]}"
|
||||
elif not finished:
|
||||
s["detail"] = f"binary exited rc={rc} before finishing all tests"
|
||||
s["passed"] = s["total"] - s["failed"]
|
||||
s["ok"] = finished and s["failed"] == 0 and s["total"] > 0
|
||||
return s
|
||||
|
||||
|
||||
# ── Go ────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def go_all_suites(repo, work):
|
||||
"""Run Go test suites across all project modules and aggregate results."""
|
||||
modules = [
|
||||
(os.path.join(repo, "Test/E2E/suite/client"),
|
||||
"Go (chain-client)"),
|
||||
(os.path.join(repo, "Common/Client/go"),
|
||||
"Go (common udpsprotocol + wshub)"),
|
||||
(os.path.join(repo, "Client/debugger"),
|
||||
"Go (debugger)"),
|
||||
]
|
||||
total_pct = 0.0
|
||||
pct_count = 0
|
||||
suites = []
|
||||
for mod_dir, name in modules:
|
||||
s = {"name": name, "lang": "go", "total": 0, "passed": 0,
|
||||
"failed": 0, "skipped": 0, "time_s": 0.0, "ok": False, "avail": False}
|
||||
cov_p = os.path.join(work, f"go_cover_{name.replace(' ', '_')}.out")
|
||||
rc, out, err = _run(
|
||||
["go", "test", "-json", f"-coverprofile={cov_p}", "./..."],
|
||||
cwd=mod_dir)
|
||||
if rc == 127:
|
||||
s["detail"] = "go toolchain not found"
|
||||
suites.append(s)
|
||||
continue
|
||||
s["avail"] = True
|
||||
cov_pct = _parse_go_json(out, s)
|
||||
if cov_pct is not None:
|
||||
s["cov_pct"] = cov_pct
|
||||
total_pct += cov_pct
|
||||
pct_count += 1
|
||||
suites.append(s)
|
||||
return suites, (round(total_pct / pct_count, 1) if pct_count else None)
|
||||
|
||||
|
||||
def _parse_go_json(out, s):
|
||||
"""Parse Go test -json output into passed/failed/skipped counts.
|
||||
Returns coverage percentage (float or None)."""
|
||||
cov_pct = None
|
||||
for line in out.splitlines():
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
act = ev.get("Action")
|
||||
if act == "pass" and ev.get("Test"):
|
||||
s["passed"] += 1
|
||||
s["total"] += 1
|
||||
elif act == "fail" and ev.get("Test"):
|
||||
s["failed"] += 1
|
||||
s["total"] += 1
|
||||
elif act == "skip" and ev.get("Test"):
|
||||
s["skipped"] += 1
|
||||
s["total"] += 1
|
||||
elif act == "output":
|
||||
m = re.search(r"coverage:\s+([\d.]+)%", ev.get("Output", ""))
|
||||
if m:
|
||||
cov_pct = float(m.group(1))
|
||||
s["ok"] = s["failed"] == 0 and s["total"] > 0
|
||||
return cov_pct
|
||||
|
||||
|
||||
# ── Python ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def py_suite(chain_dir, work):
|
||||
s = {"name": "Python (framework)", "lang": "python", "total": 0, "passed": 0,
|
||||
"failed": 0, "skipped": 0, "time_s": 0.0, "ok": False, "avail": True}
|
||||
cov_json = os.path.join(work, "py_cover.json")
|
||||
env = dict(os.environ)
|
||||
env["COVERAGE_FILE"] = os.path.join(work, ".coverage")
|
||||
have_cov = _run(["coverage", "--version"])[0] == 0
|
||||
if have_cov:
|
||||
cmd = ["coverage", "run", f"--source={chain_dir}", "-m", "unittest",
|
||||
"tests_py", "-v"]
|
||||
else:
|
||||
cmd = [sys.executable, "-m", "unittest", "tests_py", "-v"]
|
||||
rc, out, err = _run(cmd, cwd=chain_dir, env=env)
|
||||
text = out + "\n" + err
|
||||
m = re.search(r"Ran (\d+) tests? in ([\d.]+)s", text)
|
||||
if m:
|
||||
s["total"] = int(m.group(1))
|
||||
s["time_s"] = float(m.group(2))
|
||||
fm = re.search(r"failures=(\d+)", text)
|
||||
em = re.search(r"errors=(\d+)", text)
|
||||
sm = re.search(r"skipped=(\d+)", text)
|
||||
s["failed"] = (int(fm.group(1)) if fm else 0) + (int(em.group(1)) if em else 0)
|
||||
s["skipped"] = int(sm.group(1)) if sm else 0
|
||||
s["passed"] = s["total"] - s["failed"] - s["skipped"]
|
||||
s["ok"] = s["failed"] == 0 and s["total"] > 0
|
||||
pct = None
|
||||
if have_cov:
|
||||
_run(["coverage", "json", "-o", cov_json], cwd=chain_dir, env=env)
|
||||
if os.path.exists(cov_json):
|
||||
try:
|
||||
cj = json.load(open(cov_json))
|
||||
pct = round(cj["totals"]["percent_covered"], 1)
|
||||
except (KeyError, ValueError):
|
||||
pass
|
||||
s["cov_pct"] = pct
|
||||
return s
|
||||
|
||||
|
||||
# ── C++ coverage (best-effort) ────────────────────────────────────────────────
|
||||
|
||||
def _parse_lcov_info(path, repo):
|
||||
"""Parse an LCOV tracefile into per-file line/function coverage.
|
||||
|
||||
Returns (files, totals) where ``files`` is a list of
|
||||
{path (repo-relative), lines_found, lines_hit, pct, funcs_found,
|
||||
funcs_hit} sorted worst-covered first, and ``totals`` aggregates the
|
||||
same line counts across all files. Only the line counters (DA/LF/LH)
|
||||
and function counters (FNF/FNH) are read; branch data is ignored.
|
||||
"""
|
||||
files = []
|
||||
cur = None
|
||||
repo_abs = os.path.abspath(repo) + os.sep
|
||||
try:
|
||||
fh = open(path)
|
||||
except OSError:
|
||||
return [], {"lines_found": 0, "lines_hit": 0, "pct": None}
|
||||
with fh:
|
||||
for line in fh:
|
||||
line = line.rstrip("\n")
|
||||
if line.startswith("SF:"):
|
||||
src = line[3:]
|
||||
rel = src[len(repo_abs):] if src.startswith(repo_abs) else src
|
||||
cur = {"path": rel, "lines_found": 0, "lines_hit": 0,
|
||||
"funcs_found": 0, "funcs_hit": 0}
|
||||
elif cur is None:
|
||||
continue
|
||||
elif line.startswith("LF:"):
|
||||
cur["lines_found"] = int(line[3:] or 0)
|
||||
elif line.startswith("LH:"):
|
||||
cur["lines_hit"] = int(line[3:] or 0)
|
||||
elif line.startswith("FNF:"):
|
||||
cur["funcs_found"] = int(line[4:] or 0)
|
||||
elif line.startswith("FNH:"):
|
||||
cur["funcs_hit"] = int(line[4:] or 0)
|
||||
elif line == "end_of_record":
|
||||
lf = cur["lines_found"]
|
||||
cur["pct"] = round(100.0 * cur["lines_hit"] / lf, 1) if lf else None
|
||||
files.append(cur)
|
||||
cur = None
|
||||
tot_f = sum(f["lines_found"] for f in files)
|
||||
tot_h = sum(f["lines_hit"] for f in files)
|
||||
totals = {"lines_found": tot_f, "lines_hit": tot_h,
|
||||
"pct": round(100.0 * tot_h / tot_f, 1) if tot_f else None}
|
||||
files.sort(key=lambda f: (f["pct"] if f["pct"] is not None else 101.0,
|
||||
f["path"]))
|
||||
return files, totals
|
||||
|
||||
|
||||
def cpp_coverage(repo, target):
|
||||
cov = {"name": "C++", "avail": False, "pct": None, "files": [],
|
||||
"note": "not instrumented (rebuild with --cpp-coverage)"}
|
||||
build = os.path.join(repo, "Build", target)
|
||||
gcno = []
|
||||
for root, _, files in os.walk(build):
|
||||
for fn in files:
|
||||
if fn.endswith(".gcno"):
|
||||
gcno.append(os.path.join(root, fn))
|
||||
if not gcno:
|
||||
return cov
|
||||
# lcov 2.x is strict by default; tolerate the benign mismatches that arise
|
||||
# from mixing instrumented project objects with non-instrumented MARTe2/STL
|
||||
# headers, and never let a single bad file abort the whole capture.
|
||||
ign = ["--ignore-errors",
|
||||
"mismatch,source,gcov,unused,empty,negative,unsupported,inconsistent"]
|
||||
raw = os.path.join(build, "coverage_raw.info")
|
||||
rc, out, err = _run(["lcov", "--capture", "--directory", build,
|
||||
"--output-file", raw, "--quiet"] + ign, timeout=900)
|
||||
if rc != 0 or not os.path.exists(raw):
|
||||
cov["note"] = "lcov capture failed: " + (err or out or "")[-160:]
|
||||
return cov
|
||||
# Keep only this repo's own sources so the number reflects project code,
|
||||
# not the MARTe2 framework headers dragged in by templates/inlines.
|
||||
info = os.path.join(build, "coverage.info")
|
||||
rc2, _, e2 = _run(["lcov", "--extract", raw,
|
||||
os.path.join(repo, "Source", "*"),
|
||||
os.path.join(repo, "Test", "*"),
|
||||
"--output-file", info, "--quiet"] + ign, timeout=300)
|
||||
summ_file = info if (rc2 == 0 and os.path.exists(info)) else raw
|
||||
# Parse the tracefile directly for per-file detail; this also yields the
|
||||
# aggregate so the headline number and the per-file table are consistent.
|
||||
fdetail, totals = _parse_lcov_info(summ_file, repo)
|
||||
if totals["pct"] is not None:
|
||||
cov.update(avail=True, pct=totals["pct"], files=fdetail,
|
||||
lines_found=totals["lines_found"],
|
||||
lines_hit=totals["lines_hit"],
|
||||
note="lcov (project sources)" if summ_file == info else "lcov")
|
||||
return cov
|
||||
# Fall back to lcov --summary if the tracefile had no parseable line data.
|
||||
rc3, summ, _ = _run(["lcov", "--summary", summ_file] + ign)
|
||||
m = re.search(r"lines[.\s]+:\s+([\d.]+)%", summ)
|
||||
if m:
|
||||
cov.update(avail=True, pct=float(m.group(1)),
|
||||
note="lcov (project sources)" if summ_file == info else "lcov")
|
||||
else:
|
||||
cov["note"] = "lcov summary unparsed"
|
||||
return cov
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description="Collect unit tests + coverage")
|
||||
p.add_argument("--repo", required=True)
|
||||
p.add_argument("--target", default="x86-linux")
|
||||
p.add_argument("--out", required=True)
|
||||
p.add_argument("--work", default=None,
|
||||
help="scratch dir for *.xml/cover files (default: --out)")
|
||||
p.add_argument("--cpp-coverage", action="store_true")
|
||||
args = p.parse_args()
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
work = args.work or args.out
|
||||
os.makedirs(work, exist_ok=True)
|
||||
chain_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
gtest_bin = os.path.join(args.repo, "Build", args.target, "GTest", "MainGTest.ex")
|
||||
# BUILD_DIR for Test/Integration doubles the last path component
|
||||
# ($(PACKAGE)/$(lastword of CURDIR)) — see MakeStdLibDefs.gcc.
|
||||
integration_bin = os.path.join(args.repo, "Build", args.target,
|
||||
"Test", "Integration", "Integration",
|
||||
"IntegrationTests.ex")
|
||||
|
||||
go_suites, go_avg_cov = go_all_suites(args.repo, work)
|
||||
suites = ([gtest_suite(gtest_bin, work), integration_suite(integration_bin, work)]
|
||||
+ go_suites + [py_suite(chain_dir, work)])
|
||||
totals = {k: sum(s.get(k, 0) for s in suites)
|
||||
for k in ("total", "passed", "failed", "skipped")}
|
||||
ut = {"suites": suites, "totals": totals,
|
||||
"ok": all(s["ok"] for s in suites if s["avail"])}
|
||||
with open(os.path.join(args.out, "unit_tests.json"), "w") as f:
|
||||
json.dump(ut, f, indent=2)
|
||||
|
||||
langs = []
|
||||
py = next(s for s in suites if s["lang"] == "python")
|
||||
langs.append({"name": "Python", "avail": py.get("cov_pct") is not None,
|
||||
"pct": py.get("cov_pct"), "note": "coverage.py"})
|
||||
langs.append({"name": "Go", "avail": go_avg_cov is not None,
|
||||
"pct": go_avg_cov, "note": "go test -cover (avg across modules)"})
|
||||
cpp = cpp_coverage(args.repo, args.target) if args.cpp_coverage else \
|
||||
{"name": "C++", "avail": False, "pct": None, "note": "skipped (use --cpp-coverage)"}
|
||||
langs.append(cpp)
|
||||
with open(os.path.join(args.out, "coverage.json"), "w") as f:
|
||||
json.dump({"languages": langs}, f, indent=2)
|
||||
|
||||
print(f"unit_tests: {totals['passed']}/{totals['total']} pass "
|
||||
f"({totals['failed']} fail, {totals['skipped']} skip)")
|
||||
print("coverage: " + ", ".join(
|
||||
f"{l['name']}={l['pct']}%" if l["pct"] is not None else f"{l['name']}=n/a"
|
||||
for l in langs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,235 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
gen_cfg.py — MARTe2 app + StreamHub config generator for the streaming-chain E2E.
|
||||
|
||||
Given a scenario (scenarios.py) it produces two config files:
|
||||
|
||||
* MARTe app cfg: ``LinuxTimer + FileReader(input) -> IOGAM -> UDPStreamer`` per
|
||||
source. When ``oracle in {fed,both}`` a second IOGAM branch taps the same fed
|
||||
signals into a ``FileWriter`` (the "fed reference").
|
||||
* StreamHub cfg: one ``Source`` per UDPStreamer (unicast or multicast) on the
|
||||
scenario's ``ws_port``.
|
||||
|
||||
The producer rate (LinuxTimer Frequency) defaults to 1 kHz; the FileReader uses
|
||||
``EOF = "Rewind"`` so the NUM_ROWS-row input loops for the whole run. The
|
||||
validator reconstructs truth as the cyclic ground-truth sequence, so the wrap is
|
||||
expected, not an error.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import scenarios as S # noqa: E402
|
||||
|
||||
PRODUCER_HZ = 1000 # LinuxTimer frequency (Hz)
|
||||
|
||||
|
||||
def _ndims(elements):
|
||||
return 0 if elements == 1 else 1
|
||||
|
||||
|
||||
def _gam_sig(sig, datasource):
|
||||
"""A GAM signal entry referencing a DataSource (no UDPStreamer extras)."""
|
||||
return _gam_sig_named(sig["name"], sig, datasource)
|
||||
|
||||
|
||||
def _gam_sig_named(name, sig, datasource):
|
||||
"""A GAM signal entry with an explicit (possibly renamed) signal name.
|
||||
|
||||
IOGAM copies inputs to outputs positionally, so the input and output names
|
||||
may differ; we exploit that to route through the DDB with source-prefixed
|
||||
names that never clash with the TimerGAM's Counter/Time or across sources.
|
||||
"""
|
||||
return (f"{name} = {{ Type = {sig['type']} "
|
||||
f"NumberOfDimensions = {_ndims(sig['elements'])} "
|
||||
f"NumberOfElements = {sig['elements']} DataSource = {datasource} }}")
|
||||
|
||||
|
||||
def _streamer_sig(sig):
|
||||
"""A UDPStreamer DataSource signal entry (carries time/quant/unit options)."""
|
||||
parts = [f"Type = {sig['type']}",
|
||||
f"NumberOfDimensions = {_ndims(sig['elements'])}",
|
||||
f"NumberOfElements = {sig['elements']}"]
|
||||
if sig["time_mode"] and sig["time_mode"] != "PacketTime":
|
||||
parts.append(f'TimeMode = "{sig["time_mode"]}"')
|
||||
if sig["time_signal"]:
|
||||
parts.append(f'TimeSignal = "{sig["time_signal"]}"')
|
||||
if sig["sampling_rate"]:
|
||||
parts.append(f"SamplingRate = {sig['sampling_rate']}")
|
||||
if sig["quant"] and sig["quant"] != "none":
|
||||
parts.append(f'QuantizedType = "{sig["quant"]}"')
|
||||
parts.append(f"RangeMin = {sig['range_min']}")
|
||||
parts.append(f"RangeMax = {sig['range_max']}")
|
||||
if sig["unit"]:
|
||||
parts.append(f'Unit = "{sig["unit"]}"')
|
||||
return f"{sig['name']} = {{ {' '.join(parts)} }}"
|
||||
|
||||
|
||||
def _streamer_block(src, scenario):
|
||||
s = src["signals"]
|
||||
parts = [f"Port = {src['udp_port']}",
|
||||
f"MaxPayloadSize = {scenario['max_payload']}",
|
||||
f'PublishingMode = "{scenario["publishing"]}"']
|
||||
if scenario["publishing"] == "Accumulate":
|
||||
parts.append(f"MinRefreshRate = {scenario['min_refresh_hz']}")
|
||||
if scenario["publishing"] == "Decimate":
|
||||
parts.append(f"Ratio = {scenario['ratio']}")
|
||||
if scenario["network"] == "multicast":
|
||||
parts.append(f'MulticastGroup = "{src["multicast_group"]}"')
|
||||
parts.append(f"DataPort = {src['data_port']}")
|
||||
sigs = " ".join(_streamer_sig(sig) for sig in s)
|
||||
return (f" +Streamer_{src['id']} = {{ Class = UDPStreamer "
|
||||
f"{' '.join(parts)} Signals = {{ {sigs} }} }}")
|
||||
|
||||
|
||||
def write_marte_cfg(scenario, path, input_bin, tap_bin=None):
|
||||
"""Write the MARTe app cfg. ``input_bin`` is the src[0] file; additional
|
||||
sources read ``<input_bin>.<srcid>`` (matching gen_data.write_input)."""
|
||||
os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
|
||||
srcs = scenario["sources"]
|
||||
want_tap = scenario["oracle"] in ("fed", "both") and tap_bin is not None
|
||||
|
||||
_row_dt, _num_rows, producer_hz, _loop_hz = S.geometry(scenario)
|
||||
|
||||
gams = [] # +Functions entries
|
||||
datas = [] # +Data entries
|
||||
thread_funcs = ["TimerGAM"]
|
||||
|
||||
# Timer GAM (drives the schedule)
|
||||
gams.append(
|
||||
" +TimerGAM = { Class = IOGAM "
|
||||
"InputSignals = { Counter = { DataSource = ReaderTimer Type = uint32 } "
|
||||
f"Time = {{ Frequency = {producer_hz} DataSource = ReaderTimer Type = uint32 }} }} "
|
||||
"OutputSignals = { Counter = { DataSource = DDB Type = uint32 } "
|
||||
"Time = { DataSource = DDB Type = uint32 } } }")
|
||||
|
||||
for i, src in enumerate(srcs):
|
||||
sid = src["id"]
|
||||
fpath = input_bin if i == 0 else f"{input_bin}.{sid}"
|
||||
rds = f"FileReaderDS_{sid}"
|
||||
# The FileReader DataSource allows exactly one consuming Function, so a
|
||||
# tapped source must route through the DDB: ReaderGAM copies FileReader
|
||||
# -> DDB (source-prefixed names), then StreamGAM and TapGAM both read DDB.
|
||||
tap_here = want_tap and i == 0
|
||||
if tap_here:
|
||||
in_sigs = " ".join(_gam_sig(sig, rds) for sig in src["signals"])
|
||||
ddb_out = " ".join(_gam_sig_named(f"{sid}_{sig['name']}", sig, "DDB")
|
||||
for sig in src["signals"])
|
||||
gams.append(
|
||||
f" +ReaderGAM_{sid} = {{ Class = IOGAM "
|
||||
f"InputSignals = {{ {in_sigs} }} OutputSignals = {{ {ddb_out} }} }}")
|
||||
ddb_in = " ".join(_gam_sig_named(f"{sid}_{sig['name']}", sig, "DDB")
|
||||
for sig in src["signals"])
|
||||
stream_out = " ".join(_gam_sig(sig, f"Streamer_{sid}")
|
||||
for sig in src["signals"])
|
||||
gams.append(
|
||||
f" +StreamGAM_{sid} = {{ Class = IOGAM "
|
||||
f"InputSignals = {{ {ddb_in} }} OutputSignals = {{ {stream_out} }} }}")
|
||||
thread_funcs.append(f"ReaderGAM_{sid}")
|
||||
thread_funcs.append(f"StreamGAM_{sid}")
|
||||
else:
|
||||
in_sigs = " ".join(_gam_sig(sig, rds) for sig in src["signals"])
|
||||
out_sigs = " ".join(_gam_sig(sig, f"Streamer_{sid}") for sig in src["signals"])
|
||||
gams.append(
|
||||
f" +ReaderGAM_{sid} = {{ Class = IOGAM "
|
||||
f"InputSignals = {{ {in_sigs} }} OutputSignals = {{ {out_sigs} }} }}")
|
||||
thread_funcs.append(f"ReaderGAM_{sid}")
|
||||
datas.append(
|
||||
f' +{rds} = {{ Class = FileReader Filename = "{fpath}" '
|
||||
f'Interpolate = "no" FileFormat = "binary" EOF = "Rewind" }}')
|
||||
datas.append(_streamer_block(src, scenario))
|
||||
|
||||
if want_tap:
|
||||
# tap the first source's signals (now in the DDB) to a FileWriter.
|
||||
src = srcs[0]
|
||||
sid = src["id"]
|
||||
tap_in = " ".join(_gam_sig_named(f"{sid}_{sig['name']}", sig, "DDB")
|
||||
for sig in src["signals"])
|
||||
tap_out = " ".join(_gam_sig(sig, "TapWriterDS") for sig in src["signals"])
|
||||
gams.append(
|
||||
f" +TapGAM = {{ Class = IOGAM "
|
||||
f"InputSignals = {{ {tap_in} }} OutputSignals = {{ {tap_out} }} }}")
|
||||
thread_funcs.append("TapGAM")
|
||||
tap_sigs = " ".join(_gam_sig(sig, "TapWriterDS").replace(
|
||||
" DataSource = TapWriterDS", "") for sig in src["signals"])
|
||||
datas.append(
|
||||
f' +TapWriterDS = {{ Class = FileWriter NumberOfBuffers = 10 '
|
||||
f'CPUMask = 0x4 StackSize = 10000000 Filename = "{tap_bin}" '
|
||||
f'Overwrite = "yes" StoreOnTrigger = 0 FileFormat = "binary" '
|
||||
f"Signals = {{ {tap_sigs} }} }}")
|
||||
|
||||
funcs_block = "\n".join(gams)
|
||||
data_block = "\n".join(datas)
|
||||
thread_list = " ".join(thread_funcs)
|
||||
|
||||
cfg = f"""$ChainE2E = {{
|
||||
Class = RealTimeApplication
|
||||
+Functions = {{
|
||||
Class = ReferenceContainer
|
||||
{funcs_block}
|
||||
}}
|
||||
+Data = {{
|
||||
Class = ReferenceContainer DefaultDataSource = DDB
|
||||
+DDB = {{ Class = GAMDataSource }}
|
||||
+ReaderTimer = {{ Class = LinuxTimer SleepNature = "Default" Signals = {{ Counter = {{ Type = uint32 }} Time = {{ Type = uint32 }} }} }}
|
||||
{data_block}
|
||||
+Timings = {{ Class = TimingDataSource }}
|
||||
}}
|
||||
+States = {{ Class = ReferenceContainer +Running = {{ Class = RealTimeState +Threads = {{ Class = ReferenceContainer +ReaderThread = {{ Class = RealTimeThread CPUs = 0x1 Functions = {{{thread_list}}} }} }} }} }}
|
||||
+Scheduler = {{ Class = GAMScheduler TimingDataSource = Timings }}
|
||||
}}
|
||||
"""
|
||||
with open(path, "w") as f:
|
||||
f.write(cfg)
|
||||
return path
|
||||
|
||||
|
||||
def write_hub_cfg(scenario, path):
|
||||
os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
|
||||
src_blocks = []
|
||||
for src in scenario["sources"]:
|
||||
parts = [f'Label = "chain {src["id"]}"', 'Addr = "127.0.0.1"',
|
||||
f"Port = {src['udp_port']}"]
|
||||
if scenario["network"] == "multicast":
|
||||
parts.append(f'MulticastGroup = "{src["multicast_group"]}"')
|
||||
parts.append(f"DataPort = {src['data_port']}")
|
||||
src_blocks.append(f" {src['id']} = {{ {' '.join(parts)} }}")
|
||||
sources = "\n".join(src_blocks)
|
||||
cfg = f"""Hub = {{
|
||||
WSPort = {scenario['ws_port']}
|
||||
MaxPoints = 200000
|
||||
PushRate = 30
|
||||
MaxPushPoints = 2000
|
||||
RingTemporal = 1000000
|
||||
RingScalar = 100000
|
||||
Sources = {{
|
||||
{sources}
|
||||
}}
|
||||
}}
|
||||
"""
|
||||
with open(path, "w") as f:
|
||||
f.write(cfg)
|
||||
return path
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description="Generate MARTe + StreamHub cfgs")
|
||||
p.add_argument("--scenario", required=True)
|
||||
p.add_argument("--input", required=True, help="input_<id>.bin path")
|
||||
p.add_argument("--marte-out", required=True)
|
||||
p.add_argument("--hub-out", required=True)
|
||||
p.add_argument("--tap", default=None)
|
||||
args = p.parse_args()
|
||||
sc = next((s for s in S.SCENARIOS if s["id"] == args.scenario), None)
|
||||
if sc is None:
|
||||
print(f"unknown scenario {args.scenario}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
write_marte_cfg(sc, args.marte_out, args.input, args.tap)
|
||||
write_hub_cfg(sc, args.hub_out)
|
||||
print(f"marte: {args.marte_out}")
|
||||
print(f"hub: {args.hub_out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
gen_data.py — Deterministic typed/shaped data generator for the streaming-chain
|
||||
E2E suite.
|
||||
|
||||
For a scenario (see scenarios.py) it writes a MARTe2 FileReader-compatible binary
|
||||
``input_<id>.bin`` and returns a *ground-truth* dict the waveform validator uses
|
||||
to reconstruct the expected stream without re-deriving the layout.
|
||||
|
||||
MARTe2 binary format
|
||||
--------------------
|
||||
[u32 numSigs]
|
||||
per signal: [u16 TypeDescriptor.all][32B name (null-padded)][u32 numElements]
|
||||
then NUM_ROWS rows, each row = all signals' elements concatenated, every value
|
||||
little-endian at its native width.
|
||||
|
||||
Ground-truth dict schema (keyed "<src_id>:<sig_name>")
|
||||
------------------------------------------------------
|
||||
{
|
||||
"t": np.ndarray[float64] # intended sample time (s), flattened
|
||||
"v": np.ndarray # native-dtype values, flattened
|
||||
"dt": float # per-sample spacing (s)
|
||||
"formula": str
|
||||
"freq": float | None
|
||||
"elements": int
|
||||
"rows": int
|
||||
"type": str
|
||||
"quant": str
|
||||
"range_min": float | None
|
||||
"range_max": float | None
|
||||
"is_time": bool
|
||||
}
|
||||
|
||||
Values are the *raw native* values fed to the FileReader. Wire-side quantisation
|
||||
is performed by the UDPStreamer, not here — the validator applies the quant
|
||||
tolerance using the recorded quant/range fields.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import scenarios as S # noqa: E402 (TYPE_CODES / NP_DTYPE / SCENARIOS)
|
||||
|
||||
# Buffer geometry lives in scenarios.py so the seamless-loop constraint
|
||||
# (validate_scenario) and the data layout cannot drift apart.
|
||||
NUM_ROWS = S.NUM_ROWS # producer cycles written to the FileReader input
|
||||
ROW_DT = S.ROW_DT # seconds per producer cycle (row); 1 kHz producer
|
||||
|
||||
|
||||
def _sample_dt(sig, row_dt=ROW_DT):
|
||||
"""Per-element time spacing (s) for a signal."""
|
||||
if sig["sampling_rate"]:
|
||||
return 1.0 / sig["sampling_rate"]
|
||||
e = sig["elements"]
|
||||
return row_dt / e if e > 1 else row_dt
|
||||
|
||||
|
||||
def _sample_times(sig, row_dt=ROW_DT, num_rows=NUM_ROWS):
|
||||
"""Flattened intended sample times (s): num_rows*elements values."""
|
||||
e = sig["elements"]
|
||||
sdt = _sample_dt(sig, row_dt)
|
||||
rows = np.arange(num_rows, dtype=np.float64).reshape(-1, 1) * row_dt
|
||||
cols = np.arange(e, dtype=np.float64).reshape(1, -1) * sdt
|
||||
return (rows + cols).reshape(-1)
|
||||
|
||||
|
||||
def _values(sig, t):
|
||||
"""Compute float64 values for flattened sample times ``t``."""
|
||||
f = sig["formula"]
|
||||
e = sig["elements"]
|
||||
idx = np.arange(t.size, dtype=np.float64)
|
||||
if f == "sine":
|
||||
freq = sig["freq"] if sig["freq"] else 1.0
|
||||
return np.sin(2.0 * np.pi * freq * t)
|
||||
if f == "ramp":
|
||||
# linear in the global element index, normalised to a modest range
|
||||
return (idx % 1000.0)
|
||||
if f == "counter":
|
||||
return idx
|
||||
if f == "time_ns":
|
||||
return np.round(t * 1.0e9)
|
||||
if f == "time_us":
|
||||
return np.round(t * 1.0e6)
|
||||
raise ValueError(f"unknown formula {f!r} for {sig['name']}")
|
||||
|
||||
|
||||
def _native(sig, vals):
|
||||
"""Cast float64 values to the signal's native numpy dtype."""
|
||||
dt = S.NP_DTYPE[sig["type"]]
|
||||
if sig["type"] in S.FLOAT_TYPES:
|
||||
return vals.astype(dt)
|
||||
# integer: round then cast (deterministic, no banker's rounding surprises)
|
||||
return np.rint(vals).astype(dt)
|
||||
|
||||
|
||||
def build_ground_truth(scenario):
|
||||
"""Return {"<src>:<sig>": gt_dict} for every signal in the scenario."""
|
||||
row_dt, num_rows, _ph, _lh = S.geometry(scenario)
|
||||
gt = {}
|
||||
for src in scenario["sources"]:
|
||||
for sig in src["signals"]:
|
||||
t = _sample_times(sig, row_dt, num_rows)
|
||||
v = _native(sig, _values(sig, t))
|
||||
gt[f"{src['id']}:{sig['name']}"] = {
|
||||
"t": t, "v": v, "dt": _sample_dt(sig, row_dt),
|
||||
"formula": sig["formula"], "freq": sig["freq"],
|
||||
"elements": sig["elements"], "rows": num_rows,
|
||||
"type": sig["type"], "quant": sig["quant"],
|
||||
"range_min": sig["range_min"], "range_max": sig["range_max"],
|
||||
"is_time": sig["is_time"],
|
||||
}
|
||||
return gt
|
||||
|
||||
|
||||
def write_input(scenario, path):
|
||||
"""Write the MARTe binary for *one* source and return its ground-truth dict.
|
||||
|
||||
The MARTe FileReader reads a single flat row layout, so one input file maps
|
||||
to one source. Multi-source scenarios call this once per source with distinct
|
||||
paths (the orchestrator handles the per-source filename); here we write the
|
||||
first source by default but accept an explicit ``src`` via the scenario when
|
||||
a single source is present.
|
||||
"""
|
||||
os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
|
||||
gt = build_ground_truth(scenario)
|
||||
row_dt, num_rows, _ph, _lh = S.geometry(scenario)
|
||||
|
||||
# write each source to its own file: <path> for src[0], <path>.<srcid> else.
|
||||
written = {}
|
||||
for i, src in enumerate(scenario["sources"]):
|
||||
p = path if i == 0 else f"{path}.{src['id']}"
|
||||
_write_source_bin(src, p, row_dt, num_rows)
|
||||
written[src["id"]] = p
|
||||
gt["_files"] = written
|
||||
return gt
|
||||
|
||||
|
||||
def _write_source_bin(src, path, row_dt=ROW_DT, num_rows=NUM_ROWS):
|
||||
sigs = src["signals"]
|
||||
# per-signal native 2D arrays [num_rows, elements]
|
||||
cols = {}
|
||||
for sig in sigs:
|
||||
t = _sample_times(sig, row_dt, num_rows)
|
||||
v = _native(sig, _values(sig, t))
|
||||
cols[sig["name"]] = v.reshape(num_rows, sig["elements"])
|
||||
|
||||
with open(path, "wb") as f:
|
||||
f.write(struct.pack("<I", len(sigs)))
|
||||
for sig in sigs:
|
||||
f.write(struct.pack("<H", S.TYPE_CODES[sig["type"]]))
|
||||
name = (sig["name"] + "\0").encode()
|
||||
f.write((name + b"\0" * 32)[:32])
|
||||
f.write(struct.pack("<I", sig["elements"]))
|
||||
for r in range(num_rows):
|
||||
for sig in sigs:
|
||||
f.write(cols[sig["name"]][r].tobytes())
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description="Generate E2E chain input data")
|
||||
p.add_argument("--scenario", required=True, help="scenario id")
|
||||
p.add_argument("--out", required=True, help="output input_<id>.bin path")
|
||||
args = p.parse_args()
|
||||
sc = next((s for s in S.SCENARIOS if s["id"] == args.scenario), None)
|
||||
if sc is None:
|
||||
print(f"unknown scenario {args.scenario}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
gt = write_input(sc, args.out)
|
||||
for sid, fp in gt["_files"].items():
|
||||
print(f"{sid}: {fp} ({os.path.getsize(fp)} bytes)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,174 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
plots.py — Report figures for the streaming-chain E2E suite.
|
||||
|
||||
For a scenario it reads ``received_<id>.bin`` (RCV1), ``metrics_<id>.json`` and
|
||||
``checks_<id>.json`` from the artifact dir and writes:
|
||||
|
||||
* ``wave_<id>.png`` — received waveform(s) vs analytic sinusoid fit (truth)
|
||||
* ``trig_<id>.png`` — trigger signal with threshold + fired-trigger markers
|
||||
* ``zoom_<id>.png`` — full received signal with the requested zoom spans shaded
|
||||
|
||||
Missing inputs are handled gracefully (a placeholder note is drawn) so the
|
||||
orchestrator never aborts on a partial scenario.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt # noqa: E402
|
||||
import numpy as np # noqa: E402
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import scenarios as S # noqa: E402
|
||||
import gen_data as G # noqa: E402
|
||||
import validate_waveform as V # noqa: E402
|
||||
|
||||
|
||||
def _placeholder(path, msg):
|
||||
fig, ax = plt.subplots(figsize=(10, 3))
|
||||
ax.axis("off")
|
||||
ax.text(0.5, 0.5, msg, ha="center", va="center", fontsize=12)
|
||||
fig.savefig(path, dpi=110, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def _load(scenario, d):
|
||||
sid = scenario["id"]
|
||||
recv_p = os.path.join(d, f"received_{sid}.bin")
|
||||
checks_p = os.path.join(d, f"checks_{sid}.json")
|
||||
recv = V.read_received(recv_p) if os.path.exists(recv_p) else {}
|
||||
checks = json.load(open(checks_p)) if os.path.exists(checks_p) else {}
|
||||
return recv, checks
|
||||
|
||||
|
||||
def _data_keys(recv, gt):
|
||||
"""Non-time signal keys present in received, base-matched to ground truth."""
|
||||
out = []
|
||||
for k in sorted(recv):
|
||||
base = k.split("[")[0]
|
||||
g = gt.get(base)
|
||||
if g is None or g.get("is_time"):
|
||||
continue
|
||||
out.append((k, base, g))
|
||||
return out
|
||||
|
||||
|
||||
def plot_wave(scenario, recv, gt, path):
|
||||
keys = _data_keys(recv, gt)
|
||||
if not keys:
|
||||
_placeholder(path, f"{scenario['id']}: no received signal data")
|
||||
return path
|
||||
keys = keys[:4]
|
||||
fig, axes = plt.subplots(len(keys), 1, figsize=(12, 3.0 * len(keys)),
|
||||
squeeze=False)
|
||||
for ax, (k, base, g) in zip(axes[:, 0], keys):
|
||||
t, v = recv[k]
|
||||
if t.size == 0:
|
||||
ax.text(0.5, 0.5, f"{k}: empty", ha="center"); continue
|
||||
t0 = t[0]
|
||||
ax.plot(t - t0, v, ".", ms=3, label="received", color="tab:blue")
|
||||
if g["formula"] == "sine" and g["freq"] and t.size >= 8:
|
||||
corr, nrmse, amp = V.sine_shape(t, v, g["freq"])
|
||||
w = 2 * np.pi * g["freq"]
|
||||
A = np.column_stack([np.sin(w * t), np.cos(w * t), np.ones_like(t)])
|
||||
coef, *_ = np.linalg.lstsq(A, v, rcond=None)
|
||||
ts = np.linspace(t.min(), t.max(), 2000)
|
||||
As = np.column_stack([np.sin(w * ts), np.cos(w * ts), np.ones_like(ts)])
|
||||
ax.plot(ts - t0, As @ coef, "-", lw=1, color="tab:orange",
|
||||
label=f"fit f={g['freq']}Hz corr={corr:.4f}")
|
||||
ax.set_title(f"{k} ({g['type']}, quant={g['quant']}, {g['formula']})")
|
||||
ax.set_xlabel("t − t0 (s)")
|
||||
ax.legend(loc="upper right", fontsize=8)
|
||||
ax.grid(alpha=0.3)
|
||||
fig.suptitle(f"{scenario['id']} — received waveform vs truth")
|
||||
fig.tight_layout()
|
||||
fig.savefig(path, dpi=120, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
return path
|
||||
|
||||
|
||||
def plot_trigger(scenario, recv, checks, gt, path):
|
||||
trig = checks.get("trigger", [])
|
||||
if not trig:
|
||||
_placeholder(path, f"{scenario['id']}: no trigger checks")
|
||||
return path
|
||||
key = trig[0].get("key", "")
|
||||
if key not in recv or recv[key][0].size == 0:
|
||||
_placeholder(path, f"{scenario['id']}: trigger signal {key} not recorded")
|
||||
return path
|
||||
t, v = recv[key]
|
||||
t0 = t[0]
|
||||
thr = float(np.mean(v))
|
||||
fig, ax = plt.subplots(figsize=(12, 4))
|
||||
ax.plot(t - t0, v, "-", lw=0.7, color="tab:blue", label=key)
|
||||
ax.axhline(thr, color="gray", ls="--", lw=1, label=f"~threshold {thr:.3g}")
|
||||
seen = set()
|
||||
for tr in trig:
|
||||
if not tr.get("fired"):
|
||||
continue
|
||||
tt = tr["trigTime"] - t0
|
||||
lbl = f"{tr['edge']}/{tr['mode']}"
|
||||
ax.axvline(tt, color="tab:red", lw=1, alpha=0.6,
|
||||
label=("fired" if "fired" not in seen else None))
|
||||
seen.add("fired")
|
||||
ax.set_title(f"{scenario['id']} — trigger captures ({len(trig)} combos)")
|
||||
ax.set_xlabel("t − t0 (s)")
|
||||
ax.legend(loc="upper right", fontsize=8)
|
||||
ax.grid(alpha=0.3)
|
||||
fig.tight_layout()
|
||||
fig.savefig(path, dpi=120, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
return path
|
||||
|
||||
|
||||
def plot_zoom(scenario, recv, checks, gt, path):
|
||||
zooms = checks.get("zoom", [])
|
||||
keys = _data_keys(recv, gt)
|
||||
if not keys:
|
||||
_placeholder(path, f"{scenario['id']}: no received data for zoom")
|
||||
return path
|
||||
k, base, g = keys[0]
|
||||
t, v = recv[k]
|
||||
t0 = t[0]
|
||||
fig, ax = plt.subplots(figsize=(12, 4))
|
||||
ax.plot(t - t0, v, "-", lw=0.6, color="tab:blue", label=k)
|
||||
for zc in zooms:
|
||||
rg = zc.get("range", [0, 0])
|
||||
ax.axvspan(rg[0] - t0, rg[1] - t0, alpha=0.15, color="tab:green")
|
||||
ax.set_title(f"{scenario['id']} — zoom spans (n={len(zooms)})")
|
||||
ax.set_xlabel("t − t0 (s)")
|
||||
ax.legend(loc="upper right", fontsize=8)
|
||||
ax.grid(alpha=0.3)
|
||||
fig.tight_layout()
|
||||
fig.savefig(path, dpi=120, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
return path
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description="Generate E2E chain report plots")
|
||||
p.add_argument("--scenario", required=True)
|
||||
p.add_argument("--dir", required=True)
|
||||
args = p.parse_args()
|
||||
sc = next((s for s in S.SCENARIOS if s["id"] == args.scenario), None)
|
||||
if sc is None:
|
||||
print(f"unknown scenario {args.scenario}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
gt = G.build_ground_truth(sc)
|
||||
recv, checks = _load(sc, args.dir)
|
||||
w = plot_wave(sc, recv, gt, os.path.join(args.dir, f"wave_{sc['id']}.png"))
|
||||
tr = plot_trigger(sc, recv, checks, gt,
|
||||
os.path.join(args.dir, f"trig_{sc['id']}.png"))
|
||||
z = plot_zoom(sc, recv, checks, gt,
|
||||
os.path.join(args.dir, f"zoom_{sc['id']}.png"))
|
||||
print(w)
|
||||
print(tr)
|
||||
print(z)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
proc_perf.py — Snapshot a live process's CPU time and peak memory from /proc.
|
||||
|
||||
Usage: proc_perf.py <pid> <label> <out.json>
|
||||
|
||||
Reads /proc/<pid>/stat (utime+stime, in clock ticks) and /proc/<pid>/status
|
||||
(VmHWM = peak resident set, VmRSS = current) *while the process is alive* — peak
|
||||
RSS is only legible before the process exits, so the orchestrator calls this just
|
||||
before tearing the stack down. Emits a small JSON record:
|
||||
|
||||
{"label": ..., "avail": true, "cpu_s": float, "peak_rss_kb": int,
|
||||
"rss_kb": int, "threads": int}
|
||||
|
||||
If the process is already gone, writes {"label": ..., "avail": false}.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
def _clk_tck():
|
||||
try:
|
||||
return os.sysconf("SC_CLK_TCK") or 100
|
||||
except (ValueError, OSError):
|
||||
return 100
|
||||
|
||||
|
||||
def snapshot(pid):
|
||||
stat_p = f"/proc/{pid}/stat"
|
||||
status_p = f"/proc/{pid}/status"
|
||||
if not os.path.exists(stat_p):
|
||||
return {"avail": False}
|
||||
try:
|
||||
with open(stat_p) as f:
|
||||
raw = f.read()
|
||||
# The comm field is parenthesised and may contain spaces/parens, so split
|
||||
# on the final ')': everything after it is space-separated, with state as
|
||||
# the first token. utime/stime are overall fields 14/15 → indices 11/12
|
||||
# of the post-')' tokens (pid + comm consumed before the split).
|
||||
after = raw[raw.rindex(")") + 1:].split()
|
||||
utime = int(after[11])
|
||||
stime = int(after[12])
|
||||
cpu_s = (utime + stime) / float(_clk_tck())
|
||||
rec: dict = {"avail": True, "cpu_s": cpu_s}
|
||||
with open(status_p) as f:
|
||||
for line in f:
|
||||
if line.startswith("VmHWM:"):
|
||||
rec["peak_rss_kb"] = int(line.split()[1])
|
||||
elif line.startswith("VmRSS:"):
|
||||
rec["rss_kb"] = int(line.split()[1])
|
||||
elif line.startswith("Threads:"):
|
||||
rec["threads"] = int(line.split()[1])
|
||||
return rec
|
||||
except (OSError, ValueError, IndexError):
|
||||
return {"avail": False}
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 4:
|
||||
print("usage: proc_perf.py <pid> <label> <out.json>", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
pid, label, out = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
rec = snapshot(pid)
|
||||
rec["label"] = label
|
||||
with open(out, "w") as f:
|
||||
json.dump(rec, f)
|
||||
if rec.get("avail"):
|
||||
print(f"perf {label}: cpu={rec.get('cpu_s', 0):.2f}s "
|
||||
f"peakRSS={rec.get('peak_rss_kb', 0)/1024:.1f}MB")
|
||||
else:
|
||||
print(f"perf {label}: unavailable (process gone)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,306 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
report_build.py — Consolidate the E2E run into report_data.json (+ trend plots).
|
||||
|
||||
Inputs (paths via flags):
|
||||
* results.json — per-scenario status + waveform metrics (orchestrator)
|
||||
* perf_<id>_*.json — per-scenario CPU/peak-RSS snapshots (proc_perf.py)
|
||||
* unit_tests.json — GTest/Go/Python suite results (collect.py)
|
||||
* coverage.json — per-language coverage (collect.py)
|
||||
|
||||
Outputs (into --out):
|
||||
* report_data.json — everything the Typst template renders, including a
|
||||
``regression`` block that diffs this run's headline metrics against the
|
||||
previous entry in history.jsonl (progression ▲ / regression ▼).
|
||||
* history.jsonl — appended one line of headline metrics per run.
|
||||
* trend_*.png — pass-rate / coverage / fidelity / memory over runs.
|
||||
|
||||
Throughput is derived as recorded-samples / recording-duration. Memory is the
|
||||
peak resident set (VmHWM). All inputs are optional: a missing artifact degrades
|
||||
to nulls so a partial run still produces a report.
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt # noqa: E402
|
||||
|
||||
REC_DUR_S = 4.0 # client -dur; samples/sec denominator
|
||||
|
||||
|
||||
def _load(path, default=None):
|
||||
if path and os.path.exists(path):
|
||||
try:
|
||||
return json.load(open(path))
|
||||
except (ValueError, OSError):
|
||||
return default
|
||||
return default
|
||||
|
||||
|
||||
def _git_sha(repo):
|
||||
try:
|
||||
return subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=repo,
|
||||
capture_output=True, text=True, timeout=10).stdout.strip()
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _scenario_perf(work, sid):
|
||||
out = {}
|
||||
for role in ("hub", "marte"):
|
||||
rec = _load(os.path.join(work, f"perf_{sid}_{role}.json"), {})
|
||||
if rec and rec.get("avail"):
|
||||
out[role] = {"cpu_s": round(rec.get("cpu_s", 0.0), 3),
|
||||
"peak_rss_mb": round(rec.get("peak_rss_kb", 0) / 1024.0, 1),
|
||||
"threads": rec.get("threads")}
|
||||
return out
|
||||
|
||||
|
||||
def _scenario_descs():
|
||||
"""id -> human description, imported from the scenario matrix (best effort)."""
|
||||
try:
|
||||
from scenarios import SCENARIOS
|
||||
return {s["id"]: s.get("desc") for s in SCENARIOS}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def build_e2e(results, work):
|
||||
descs = _scenario_descs()
|
||||
scen = []
|
||||
corrs, rss_vals, cpu_vals, tput_vals = [], [], [], []
|
||||
for r in results.get("scenarios", []):
|
||||
sid = r["id"]
|
||||
metrics = r.get("metrics", {})
|
||||
sigs = []
|
||||
nrecv_total = 0
|
||||
for key, m in (metrics.get("signals", {}) or {}).items():
|
||||
nrecv_total += int(m.get("n_recv", 0) or 0)
|
||||
if "corr" in m:
|
||||
corrs.append(m["corr"])
|
||||
sigs.append({
|
||||
"key": key, "pass": m.get("pass"),
|
||||
"type": m.get("type"), "quant": m.get("quant"),
|
||||
"max_abs_err": m.get("max_abs_err"),
|
||||
"corr": m.get("corr"), "nrmse": m.get("nrmse"),
|
||||
"fidelity_ok": m.get("fidelity_ok"), "shape_ok": m.get("shape_ok"),
|
||||
"n_recv": m.get("n_recv"),
|
||||
})
|
||||
perf = _scenario_perf(work, sid)
|
||||
for role in perf.values():
|
||||
if role.get("peak_rss_mb"):
|
||||
rss_vals.append(role["peak_rss_mb"])
|
||||
if role.get("cpu_s"):
|
||||
cpu_vals.append(role["cpu_s"])
|
||||
tput = round(nrecv_total / REC_DUR_S, 1) if nrecv_total else 0.0
|
||||
if tput:
|
||||
tput_vals.append(tput)
|
||||
client = metrics.get("client", {}) or {}
|
||||
# waveform overview image (plots.py writes it into --work); record the
|
||||
# basename only when present so the Typst template can embed it without
|
||||
# tripping over a missing file (Typst read() throws on absence).
|
||||
wave_img = f"wave_{sid}.png"
|
||||
has_wave = os.path.exists(os.path.join(work, wave_img))
|
||||
scen.append({
|
||||
"id": sid, "status": r.get("status"),
|
||||
"desc": descs.get(sid),
|
||||
"known_issue": r.get("known_issue"),
|
||||
"signals": sigs, "perf": perf, "throughput_sps": tput,
|
||||
"live_frames": (client.get("live", {}) or {}).get("frames"),
|
||||
"rollup": client.get("_rollup", {}),
|
||||
# detailed client behavioural checks (chain-client checks_<id>.json),
|
||||
# surfaced so the report can show real zoom ranges + trigger captures
|
||||
# rather than only the pass/fail rollup booleans.
|
||||
"zoom": client.get("zoom", []) or [],
|
||||
"window": client.get("window", {}) or {},
|
||||
"trigger": client.get("trigger", []) or [],
|
||||
"wave_img": wave_img if has_wave else None,
|
||||
})
|
||||
agg = {
|
||||
"mean_corr": round(sum(corrs) / len(corrs), 4) if corrs else None,
|
||||
"mean_peak_rss_mb": round(sum(rss_vals) / len(rss_vals), 1) if rss_vals else None,
|
||||
"mean_cpu_s": round(sum(cpu_vals) / len(cpu_vals), 3) if cpu_vals else None,
|
||||
"mean_throughput_sps": round(sum(tput_vals) / len(tput_vals), 1) if tput_vals else None,
|
||||
}
|
||||
npass = sum(1 for s in scen if s["status"] == "PASS")
|
||||
nfail = sum(1 for s in scen if s["status"] == "FAIL")
|
||||
nskip = sum(1 for s in scen if s["status"] == "SKIP")
|
||||
nxfail = sum(1 for s in scen if s["status"] == "XFAIL")
|
||||
nxpass = sum(1 for s in scen if s["status"] == "XPASS")
|
||||
return {
|
||||
"overall": results.get("overall", "FAIL"),
|
||||
"n_pass": npass, "n_fail": nfail, "n_skip": nskip,
|
||||
"n_xfail": nxfail, "n_xpass": nxpass,
|
||||
"scenarios": scen, "agg": agg,
|
||||
}
|
||||
|
||||
|
||||
def headline(e2e, ut, cov):
|
||||
cov_by = {c["name"]: c.get("pct") for c in cov.get("languages", [])}
|
||||
t = ut.get("totals", {})
|
||||
return {
|
||||
"e2e_pass": e2e["n_pass"], "e2e_fail": e2e["n_fail"],
|
||||
"e2e_xfail": e2e.get("n_xfail", 0), "e2e_xpass": e2e.get("n_xpass", 0),
|
||||
"e2e_total": (e2e["n_pass"] + e2e["n_fail"] + e2e["n_skip"]
|
||||
+ e2e.get("n_xfail", 0) + e2e.get("n_xpass", 0)),
|
||||
"unit_pass": t.get("passed", 0), "unit_fail": t.get("failed", 0),
|
||||
"unit_total": t.get("total", 0),
|
||||
"cov_python": cov_by.get("Python"), "cov_go": cov_by.get("Go"),
|
||||
"cov_cpp": cov_by.get("C++"),
|
||||
"mean_corr": e2e["agg"]["mean_corr"],
|
||||
"mean_peak_rss_mb": e2e["agg"]["mean_peak_rss_mb"],
|
||||
"mean_cpu_s": e2e["agg"]["mean_cpu_s"],
|
||||
"mean_throughput_sps": e2e["agg"]["mean_throughput_sps"],
|
||||
}
|
||||
|
||||
|
||||
# field → "higher is better" (True), "lower is better" (False)
|
||||
_DIRECTION = {
|
||||
"e2e_pass": True, "e2e_fail": False, "unit_pass": True, "unit_fail": False,
|
||||
"cov_python": True, "cov_go": True, "cov_cpp": True, "mean_corr": True,
|
||||
"mean_peak_rss_mb": False, "mean_cpu_s": False, "mean_throughput_sps": True,
|
||||
}
|
||||
_LABELS = {
|
||||
"e2e_pass": "E2E scenarios passed", "e2e_fail": "E2E scenarios failed",
|
||||
"unit_pass": "Unit tests passed", "unit_fail": "Unit tests failed",
|
||||
"cov_python": "Python coverage %", "cov_go": "Go coverage %",
|
||||
"cov_cpp": "C++ coverage %", "mean_corr": "Mean sine corr",
|
||||
"mean_peak_rss_mb": "Mean peak RSS (MB)", "mean_cpu_s": "Mean CPU (s)",
|
||||
"mean_throughput_sps": "Mean throughput (samp/s)",
|
||||
}
|
||||
|
||||
|
||||
def regression(curr, prev):
|
||||
rows = []
|
||||
for k, label in _LABELS.items():
|
||||
c = curr.get(k)
|
||||
p = prev.get(k) if prev else None
|
||||
better = None
|
||||
delta = None
|
||||
if isinstance(c, (int, float)) and isinstance(p, (int, float)):
|
||||
delta = round(c - p, 4)
|
||||
if delta == 0:
|
||||
better = None
|
||||
else:
|
||||
better = (delta > 0) == _DIRECTION[k]
|
||||
rows.append({"name": label, "key": k, "current": c, "previous": p,
|
||||
"delta": delta, "better": better,
|
||||
"higher_better": _DIRECTION[k]})
|
||||
return rows
|
||||
|
||||
|
||||
def trend_plots(history, out):
|
||||
if not history:
|
||||
return []
|
||||
xs = list(range(len(history)))
|
||||
labels = [h.get("ts_short", str(i)) for i, h in enumerate(history)]
|
||||
made = []
|
||||
|
||||
def _plot(fname, series, title, ylabel):
|
||||
ys = [[h.get(k) for h in history] for _, k in series]
|
||||
if all(all(v is None for v in y) for y in ys):
|
||||
return
|
||||
fig, ax = plt.subplots(figsize=(7, 3))
|
||||
for (lbl, _), y in zip(series, ys):
|
||||
xp = [x for x, v in zip(xs, y) if v is not None]
|
||||
yp = [v for v in y if v is not None]
|
||||
if yp:
|
||||
ax.plot(xp, yp, "o-", label=lbl)
|
||||
ax.set_title(title)
|
||||
ax.set_ylabel(ylabel)
|
||||
ax.set_xticks(xs)
|
||||
ax.set_xticklabels(labels, rotation=45, ha="right", fontsize=7)
|
||||
ax.grid(alpha=0.3)
|
||||
ax.legend(fontsize=8)
|
||||
fig.tight_layout()
|
||||
p = os.path.join(out, fname)
|
||||
fig.savefig(p, dpi=110)
|
||||
plt.close(fig)
|
||||
made.append(p)
|
||||
|
||||
_plot("trend_tests.png",
|
||||
[("E2E pass", "e2e_pass"), ("Unit pass", "unit_pass")],
|
||||
"Passing tests over runs", "count")
|
||||
_plot("trend_coverage.png",
|
||||
[("Python", "cov_python"), ("Go", "cov_go"), ("C++", "cov_cpp")],
|
||||
"Code coverage over runs", "% covered")
|
||||
_plot("trend_fidelity.png", [("Mean sine corr", "mean_corr")],
|
||||
"Waveform fidelity over runs", "correlation")
|
||||
_plot("trend_perf.png",
|
||||
[("Peak RSS (MB)", "mean_peak_rss_mb"), ("CPU (s)", "mean_cpu_s")],
|
||||
"Resource use over runs", "value")
|
||||
return made
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Build E2E report_data.json")
|
||||
ap.add_argument("--repo", required=True)
|
||||
ap.add_argument("--results", required=True)
|
||||
ap.add_argument("--work", required=True)
|
||||
ap.add_argument("--out", required=True)
|
||||
args = ap.parse_args()
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
|
||||
results = _load(args.results, {"overall": "FAIL", "scenarios": []})
|
||||
ut = _load(os.path.join(args.out, "unit_tests.json"), {"suites": [], "totals": {}})
|
||||
cov = _load(os.path.join(args.out, "coverage.json"), {"languages": []})
|
||||
|
||||
e2e = build_e2e(results, args.work)
|
||||
now = datetime.datetime.now()
|
||||
meta = {"timestamp": now.isoformat(timespec="seconds"),
|
||||
"ts_short": now.strftime("%m-%d %H:%M"),
|
||||
"git_sha": _git_sha(args.repo), "target": "x86-linux"}
|
||||
|
||||
hl = headline(e2e, ut, cov)
|
||||
|
||||
# history: read previous, then append current
|
||||
hist_path = os.path.join(args.out, "history.jsonl")
|
||||
history = []
|
||||
if os.path.exists(hist_path):
|
||||
for line in open(hist_path):
|
||||
line = line.strip()
|
||||
if line:
|
||||
try:
|
||||
history.append(json.loads(line))
|
||||
except ValueError:
|
||||
pass
|
||||
prev = history[-1] if history else None
|
||||
reg = regression(hl, prev)
|
||||
|
||||
entry = dict(hl)
|
||||
entry["timestamp"] = meta["timestamp"]
|
||||
entry["ts_short"] = meta["ts_short"]
|
||||
entry["git_sha"] = meta["git_sha"]
|
||||
entry["overall"] = e2e["overall"]
|
||||
with open(hist_path, "a") as f:
|
||||
f.write(json.dumps(entry) + "\n")
|
||||
history.append(entry)
|
||||
|
||||
plots = [os.path.basename(p) for p in trend_plots(history, args.out)]
|
||||
|
||||
doc = {
|
||||
"meta": meta, "e2e": e2e, "unit_tests": ut,
|
||||
"coverage": cov, "regression": reg, "headline": hl,
|
||||
"trend_plots": plots, "history_len": len(history),
|
||||
"is_first_run": prev is None,
|
||||
}
|
||||
with open(os.path.join(args.out, "report_data.json"), "w") as f:
|
||||
json.dump(doc, f, indent=2)
|
||||
|
||||
print(f"report_data.json: e2e {e2e['n_pass']}/{e2e['n_pass']+e2e['n_fail']+e2e['n_skip']}"
|
||||
f" pass, units {hl['unit_pass']}/{hl['unit_total']}, "
|
||||
f"cov py={hl['cov_python']} go={hl['cov_go']} cpp={hl['cov_cpp']}")
|
||||
if prev:
|
||||
ups = sum(1 for r in reg if r["better"] is True)
|
||||
downs = sum(1 for r in reg if r["better"] is False)
|
||||
print(f"regression vs previous run: {ups} improved, {downs} regressed")
|
||||
else:
|
||||
print("regression: first run (baseline established)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+291
@@ -0,0 +1,291 @@
|
||||
#!/usr/bin/env bash
|
||||
# run_e2e.sh — Full-chain E2E orchestrator for the streaming chain
|
||||
#
|
||||
# MARTe2 app (FileReader -> IOGAM -> UDPStreamer)
|
||||
# -> UDPS -> StreamHub -> chain-client (record + zoom/window/trigger)
|
||||
# -> validate_waveform.py -> plots.py -> results.json [-> PDF]
|
||||
#
|
||||
# Per scenario (scenarios.py) it generates input data + both cfgs, runs the
|
||||
# two-process stack, drives the mock client, validates the recorded waveform
|
||||
# against the analytic/fed oracle, renders plots, and aggregates results.json.
|
||||
# Artifacts: Build/x86-linux/E2E/chain/ (report) and /tmp/chain_e2e/ (scratch).
|
||||
#
|
||||
# Usage: ./run_e2e.sh [--skip-build] [--only <id>] [--pdf-only]
|
||||
set -u
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
|
||||
TARGET=x86-linux
|
||||
BUILD_DIR="${REPO_ROOT}/Build/${TARGET}"
|
||||
OUT_DIR="${BUILD_DIR}/E2E/chain"
|
||||
WORK="/tmp/chain_e2e"
|
||||
mkdir -p "${OUT_DIR}" "${WORK}"
|
||||
|
||||
SKIP_BUILD=0
|
||||
ONLY=""
|
||||
PDF_ONLY=0
|
||||
CPP_COV=1
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--skip-build) SKIP_BUILD=1 ;;
|
||||
--only) shift; ONLY="$1" ;;
|
||||
--pdf-only) PDF_ONLY=1 ;;
|
||||
--cpp-coverage) CPP_COV=1 ;;
|
||||
--help|-h) echo "Usage: $0 [--skip-build] [--only <id>] [--pdf-only] [--cpp-coverage]"; exit 0 ;;
|
||||
*) echo "unknown arg $1" >&2; exit 2 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
ENV_SCRIPT="${REPO_ROOT}/env.sh"
|
||||
[ -f "${ENV_SCRIPT}" ] || { echo "ERROR: ${ENV_SCRIPT} not found" >&2; exit 1; }
|
||||
: "${LD_LIBRARY_PATH:=}" # env.sh appends to it under our set -u
|
||||
source "${ENV_SCRIPT}"
|
||||
|
||||
COMP="${MARTe2_Components_DIR}/Build/${TARGET}/Components"
|
||||
export LD_LIBRARY_PATH="\
|
||||
${BUILD_DIR}/Components/DataSources/UDPStreamer:\
|
||||
${BUILD_DIR}/Components/Interfaces/UDPStream:\
|
||||
${MARTe2_DIR}/Build/${TARGET}/Core:\
|
||||
${COMP}/DataSources/LinuxTimer:\
|
||||
${COMP}/DataSources/FileDataSource:\
|
||||
${COMP}/GAMs/IOGAM:\
|
||||
${LD_LIBRARY_PATH:-}"
|
||||
|
||||
MARTE_APP="${MARTe2_DIR}/Build/${TARGET}/App/MARTeApp.ex"
|
||||
STREAMHUB_EX="${BUILD_DIR}/StreamHub/StreamHub.ex"
|
||||
CLIENT="${SCRIPT_DIR}/client/chain-client"
|
||||
|
||||
PY="python3"
|
||||
SCEN() { ${PY} "${SCRIPT_DIR}/scenarios.py" >/dev/null 2>&1; }
|
||||
|
||||
# ── PDF-only shortcut (Task 9 fills in the compile) ──────────────────────────
|
||||
if [ "${PDF_ONLY}" -eq 1 ]; then
|
||||
if command -v typst >/dev/null 2>&1 && [ -f "${SCRIPT_DIR}/E2E_Report.typ" ]; then
|
||||
cp "${SCRIPT_DIR}/E2E_Report.typ" "${OUT_DIR}/" 2>/dev/null || true
|
||||
(cd "${OUT_DIR}" && typst compile E2E_Report.typ E2E_Report.pdf) \
|
||||
&& echo "PDF: ${OUT_DIR}/E2E_Report.pdf"
|
||||
else
|
||||
echo "typst or E2E_Report.typ missing — skipping PDF"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Build ────────────────────────────────────────────────────────────────────
|
||||
if [ "${SKIP_BUILD}" -eq 0 ]; then
|
||||
echo "── Building components ──"
|
||||
make -C "${REPO_ROOT}/Source/Components/Interfaces/UDPStream" -f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -1
|
||||
make -C "${REPO_ROOT}/Source/Components/DataSources/UDPStreamer" -f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -1
|
||||
make -C "${REPO_ROOT}/Source/Applications/StreamHub" -f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -1
|
||||
fi
|
||||
if [ ! -x "${CLIENT}" ]; then
|
||||
echo "── Building chain-client ──"
|
||||
(cd "${SCRIPT_DIR}/client" && go build -o chain-client .) || { echo "client build failed"; exit 1; }
|
||||
fi
|
||||
[ -x "${MARTE_APP}" ] || { echo "ERROR: MARTeApp.ex not found at ${MARTE_APP}" >&2; exit 1; }
|
||||
[ -x "${STREAMHUB_EX}" ] || { echo "ERROR: StreamHub.ex not found" >&2; exit 1; }
|
||||
|
||||
# ── Scenario list (id|ws_port|udp_port0|network|oracle|trig|checks) ──────────
|
||||
LIST="$(${PY} - "${ONLY}" <<'PY'
|
||||
import sys, os
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath("Test/E2E/suite/scenarios.py")))
|
||||
sys.path.insert(0, os.path.join(os.getcwd(), "Test/E2E/suite"))
|
||||
import scenarios as S
|
||||
only = sys.argv[1] if len(sys.argv) > 1 else ""
|
||||
for s in S.SCENARIOS:
|
||||
if only and s["id"] != only:
|
||||
continue
|
||||
trig = s.get("trig_signal") or ""
|
||||
checks = ",".join(s.get("client_checks", []))
|
||||
if not trig:
|
||||
checks = ",".join(c for c in s.get("client_checks", []) if c != "trigger")
|
||||
print("|".join([s["id"], str(s["ws_port"]), str(s["sources"][0]["udp_port"]),
|
||||
s["network"], s["oracle"], trig, checks]))
|
||||
PY
|
||||
)"
|
||||
|
||||
if [ -z "${LIST}" ]; then echo "no scenarios selected"; exit 1; fi
|
||||
|
||||
SCEN_IDS=""
|
||||
HUB_PID=""; APP_PID=""
|
||||
cleanup() {
|
||||
[ -n "${APP_PID}" ] && kill "${APP_PID}" 2>/dev/null
|
||||
[ -n "${HUB_PID}" ] && kill "${HUB_PID}" 2>/dev/null
|
||||
wait "${APP_PID}" 2>/dev/null; wait "${HUB_PID}" 2>/dev/null
|
||||
APP_PID=""; HUB_PID=""
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
while IFS='|' read -r ID WSPORT UDPPORT NET ORACLE TRIG CHECKS; do
|
||||
[ -z "${ID}" ] && continue
|
||||
SCEN_IDS="${SCEN_IDS} ${ID}"
|
||||
echo ""
|
||||
echo "══ scenario ${ID} (net=${NET} oracle=${ORACLE} ws=${WSPORT}) ══"
|
||||
: > "${WORK}/status_${ID}.txt"
|
||||
|
||||
# multicast route probe
|
||||
if [ "${NET}" = "multicast" ]; then
|
||||
GRP="$(${PY} -c "import sys;sys.path.insert(0,'${SCRIPT_DIR}');import scenarios as S;print(next(s for s in S.SCENARIOS if s['id']=='${ID}')['sources'][0]['multicast_group'])")"
|
||||
if ! ip route get "${GRP}" >/dev/null 2>&1; then
|
||||
echo " SKIP: no multicast route to ${GRP}"
|
||||
echo "SKIP" > "${WORK}/status_${ID}.txt"
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
INPUT="${WORK}/input_${ID}.bin"
|
||||
MCFG="${WORK}/m_${ID}.cfg"
|
||||
HCFG="${WORK}/h_${ID}.cfg"
|
||||
TAP=""
|
||||
if [ "${ORACLE}" = "fed" ] || [ "${ORACLE}" = "both" ]; then
|
||||
TAP="${WORK}/tap_${ID}.bin"
|
||||
fi
|
||||
|
||||
${PY} "${SCRIPT_DIR}/gen_data.py" --scenario "${ID}" --out "${INPUT}" || { echo FAIL > "${WORK}/status_${ID}.txt"; continue; }
|
||||
if [ -n "${TAP}" ]; then
|
||||
${PY} "${SCRIPT_DIR}/gen_cfg.py" --scenario "${ID}" --input "${INPUT}" --marte-out "${MCFG}" --hub-out "${HCFG}" --tap "${TAP}"
|
||||
else
|
||||
${PY} "${SCRIPT_DIR}/gen_cfg.py" --scenario "${ID}" --input "${INPUT}" --marte-out "${MCFG}" --hub-out "${HCFG}"
|
||||
fi
|
||||
|
||||
HUB_LOG="${OUT_DIR}/hub_${ID}.log"
|
||||
APP_LOG="${OUT_DIR}/marte_${ID}.log"
|
||||
rm -f "${WORK}/received_${ID}.bin" "${WORK}/checks_${ID}.json" "${WORK}/metrics_${ID}.json"
|
||||
|
||||
"${STREAMHUB_EX}" -cfg "${HCFG}" > "${HUB_LOG}" 2>&1 &
|
||||
HUB_PID=$!
|
||||
sleep 1
|
||||
timeout 120 "${MARTE_APP}" -l RealTimeLoader -f "${MCFG}" -s Running > "${APP_LOG}" 2>&1 &
|
||||
APP_PID=$!
|
||||
sleep 1
|
||||
# APP_PID is the `timeout` wrapper; perf must target the real MARTeApp child.
|
||||
APP_PERF_PID="$(pgrep -P "${APP_PID}" 2>/dev/null | head -1)"
|
||||
[ -z "${APP_PERF_PID}" ] && APP_PERF_PID="${APP_PID}"
|
||||
|
||||
TRIGARG=""
|
||||
[ -n "${TRIG}" ] && TRIGARG="-trigsig ${TRIG}"
|
||||
if "${CLIENT}" -hub "127.0.0.1:${WSPORT}" -scenario "${ID}" ${TRIGARG} \
|
||||
-checks "${CHECKS}" -out "${WORK}" -dur 4 > "${OUT_DIR}/client_${ID}.log" 2>&1; then
|
||||
echo " client OK"
|
||||
else
|
||||
echo " client FAILED (see client_${ID}.log)"
|
||||
tail -3 "${OUT_DIR}/client_${ID}.log" | sed 's/^/ /'
|
||||
fi
|
||||
|
||||
# Snapshot CPU/peak-RSS while the stack is still alive (peak RSS is only
|
||||
# legible before exit), then tear it down.
|
||||
[ -n "${HUB_PID}" ] && ${PY} "${SCRIPT_DIR}/proc_perf.py" "${HUB_PID}" streamhub "${WORK}/perf_${ID}_hub.json" || true
|
||||
[ -n "${APP_PERF_PID}" ] && ${PY} "${SCRIPT_DIR}/proc_perf.py" "${APP_PERF_PID}" marte "${WORK}/perf_${ID}_marte.json" || true
|
||||
cleanup
|
||||
|
||||
# validate + plot
|
||||
VARGS="--scenario ${ID} --received ${WORK}/received_${ID}.bin --checks ${WORK}/checks_${ID}.json --out ${WORK}/metrics_${ID}.json"
|
||||
[ -n "${TAP}" ] && [ -f "${TAP}" ] && VARGS="${VARGS} --tap ${TAP}"
|
||||
if ${PY} "${SCRIPT_DIR}/validate_waveform.py" ${VARGS}; then
|
||||
echo "PASS" > "${WORK}/status_${ID}.txt"
|
||||
else
|
||||
echo "FAIL" > "${WORK}/status_${ID}.txt"
|
||||
fi
|
||||
${PY} "${SCRIPT_DIR}/plots.py" --scenario "${ID}" --dir "${WORK}" >/dev/null 2>&1 || true
|
||||
done <<< "${LIST}"
|
||||
|
||||
trap - EXIT
|
||||
cleanup
|
||||
|
||||
# ── Aggregate results.json ───────────────────────────────────────────────────
|
||||
# Scenarios carrying a `known_issue` marker exercise a documented, not-yet-fixed
|
||||
# chain gap: a raw FAIL is reclassified XFAIL (expected failure — does not break
|
||||
# the green baseline) and a raw PASS becomes XPASS (the bug was unexpectedly
|
||||
# fixed; the marker should be dropped). Overall is PASS when nothing FAILs and
|
||||
# nothing unexpectedly XPASSes.
|
||||
WORK="${WORK}" OUT_DIR="${OUT_DIR}" SCEN_IDS="${SCEN_IDS}" \
|
||||
SCRIPT_DIR="${SCRIPT_DIR}" ${PY} - <<'PY'
|
||||
import json, os, sys
|
||||
work = os.environ["WORK"]; out = os.environ["OUT_DIR"]
|
||||
ids = os.environ["SCEN_IDS"].split()
|
||||
sys.path.insert(0, os.environ["SCRIPT_DIR"])
|
||||
try:
|
||||
from scenarios import SCENARIOS
|
||||
known = {s["id"]: s.get("known_issue") for s in SCENARIOS}
|
||||
except Exception:
|
||||
known = {}
|
||||
results = []
|
||||
for sid in ids:
|
||||
rec = {"id": sid}
|
||||
st = os.path.join(work, f"status_{sid}.txt")
|
||||
raw = open(st).read().strip() if os.path.exists(st) else "UNKNOWN"
|
||||
ki = known.get(sid)
|
||||
if ki:
|
||||
rec["known_issue"] = ki
|
||||
if raw == "FAIL":
|
||||
raw = "XFAIL" # expected failure — documented chain gap
|
||||
elif raw == "PASS":
|
||||
raw = "XPASS" # unexpectedly fixed — drop the marker
|
||||
rec["status"] = raw
|
||||
mp = os.path.join(work, f"metrics_{sid}.json")
|
||||
if os.path.exists(mp):
|
||||
rec["metrics"] = json.load(open(mp))
|
||||
results.append(rec)
|
||||
# Green when no hard FAIL and no XPASS (an XPASS means a known_issue marker is
|
||||
# now stale and must be removed — surfaced as a failure to force the cleanup).
|
||||
overall = (bool(results)
|
||||
and all(r["status"] in ("PASS", "SKIP", "XFAIL") for r in results))
|
||||
doc = {"overall": "PASS" if overall else "FAIL", "scenarios": results}
|
||||
with open(os.path.join(out, "results.json"), "w") as f:
|
||||
json.dump(doc, f, indent=2)
|
||||
c = lambda st: sum(r["status"] == st for r in results)
|
||||
print(f"\nresults.json: {c('PASS')} pass, {c('FAIL')} fail, {c('SKIP')} skip, "
|
||||
f"{c('XFAIL')} xfail, {c('XPASS')} xpass → {doc['overall']}")
|
||||
PY
|
||||
|
||||
# ── Unit tests + coverage ────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "── Unit tests + coverage ──"
|
||||
|
||||
# Optional C++ line coverage: rebuild the project's libraries + GTest with gcov
|
||||
# instrumentation in place (OPTIM/LFLAGS are the MARTe2-sanctioned override
|
||||
# hooks), let collect.py run the instrumented GTest (emits .gcda) and lcov, then
|
||||
# restore a clean build so later --skip-build runs aren't left instrumented.
|
||||
CPP_COV_FLAG=""
|
||||
if [ "${CPP_COV}" -eq 1 ]; then
|
||||
echo " building instrumented (gcov) libraries + GTest ..."
|
||||
COV_O="--coverage"
|
||||
COV_L="-Wl,--no-as-needed -fPIC --coverage"
|
||||
make -C "${REPO_ROOT}" -f Makefile.gcc clean >/dev/null 2>&1 || true
|
||||
make -C "${REPO_ROOT}" -f Makefile.gcc core TARGET="${TARGET}" \
|
||||
OPTIM="${COV_O}" LFLAGS="${COV_L}" 2>&1 | tail -1
|
||||
for d in Test/Components/DataSources/UDPStreamer Test/Components/DataSources/UDPStreamerClient Test/Applications/StreamHub Test/GTest Test/Integration; do
|
||||
make -C "${REPO_ROOT}/${d}" -f Makefile.gcc TARGET="${TARGET}" \
|
||||
OPTIM="${COV_O}" LFLAGS="${COV_L}" 2>&1 | tail -1
|
||||
done
|
||||
CPP_COV_FLAG="--cpp-coverage"
|
||||
fi
|
||||
|
||||
${PY} "${SCRIPT_DIR}/collect.py" --repo "${REPO_ROOT}" --target "${TARGET}" \
|
||||
--out "${OUT_DIR}" --work "${WORK}" ${CPP_COV_FLAG} || true
|
||||
|
||||
if [ "${CPP_COV}" -eq 1 ]; then
|
||||
echo " restoring non-instrumented build ..."
|
||||
make -C "${REPO_ROOT}" -f Makefile.gcc clean >/dev/null 2>&1 || true
|
||||
make -C "${REPO_ROOT}" -f Makefile.gcc core apps TARGET="${TARGET}" 2>&1 | tail -1
|
||||
(cd "${SCRIPT_DIR}/client" && go build -o chain-client . >/dev/null 2>&1) || true
|
||||
fi
|
||||
|
||||
# ── Consolidated report data (+ history/regression + trend plots) ────────────
|
||||
${PY} "${SCRIPT_DIR}/report_build.py" --repo "${REPO_ROOT}" \
|
||||
--results "${OUT_DIR}/results.json" --work "${WORK}" --out "${OUT_DIR}" || true
|
||||
|
||||
# ── PDF ──────────────────────────────────────────────────────────────────────
|
||||
if command -v typst >/dev/null 2>&1 && [ -f "${SCRIPT_DIR}/E2E_Report.typ" ]; then
|
||||
cp "${SCRIPT_DIR}/E2E_Report.typ" "${OUT_DIR}/" 2>/dev/null || true
|
||||
# Embedded waveform overviews live in WORK; the template references them by
|
||||
# bare name, so stage them next to report_data.json before compiling.
|
||||
cp "${WORK}"/wave_*.png "${OUT_DIR}/" 2>/dev/null || true
|
||||
if (cd "${OUT_DIR}" && typst compile E2E_Report.typ E2E_Report.pdf); then
|
||||
echo "PDF: ${OUT_DIR}/E2E_Report.pdf"
|
||||
else
|
||||
echo "typst compile failed (report_data.json present at ${OUT_DIR})"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Done — artifacts in ${OUT_DIR} and ${WORK}"
|
||||
Executable
+85
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env bash
|
||||
# run_stress.sh — Capacity/stress harness for the streaming chain.
|
||||
#
|
||||
# MARTe2 app (FileReader -> IOGAM -> UDPStreamer)
|
||||
# -> UDPS -> StreamHub(s) -> chain-client(s) (stress: liveness + zoom latency)
|
||||
# -> proc_perf (cpu/peakRSS) -> per-case gate -> stress_results.json
|
||||
#
|
||||
# It sweeps one load axis at a time (signal size/count, subscriber fan-out, source
|
||||
# count, WS-client count, zoom request rate — see stress.py) and gates each case on
|
||||
# survival + liveness (hard) and RSS + zoom-p95 latency (soft). This is the
|
||||
# capacity sibling of run_e2e.sh (which gates waveform correctness).
|
||||
#
|
||||
# Usage: ./run_stress.sh [--skip-build] [--only <id>] [--axis <axis>]
|
||||
set -u
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
|
||||
TARGET=x86-linux
|
||||
BUILD_DIR="${REPO_ROOT}/Build/${TARGET}"
|
||||
OUT_DIR="${BUILD_DIR}/E2E/chain/stress"
|
||||
WORK="/tmp/chain_stress"
|
||||
mkdir -p "${OUT_DIR}" "${WORK}"
|
||||
|
||||
SKIP_BUILD=0
|
||||
ONLY=""
|
||||
AXIS=""
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--skip-build) SKIP_BUILD=1 ;;
|
||||
--only) shift; ONLY="$1" ;;
|
||||
--axis) shift; AXIS="$1" ;;
|
||||
--help|-h) echo "Usage: $0 [--skip-build] [--only <id>] [--axis <axis>]"; exit 0 ;;
|
||||
*) echo "unknown arg $1" >&2; exit 2 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
ENV_SCRIPT="${REPO_ROOT}/env.sh"
|
||||
[ -f "${ENV_SCRIPT}" ] || { echo "ERROR: ${ENV_SCRIPT} not found" >&2; exit 1; }
|
||||
: "${LD_LIBRARY_PATH:=}"
|
||||
source "${ENV_SCRIPT}"
|
||||
|
||||
COMP="${MARTe2_Components_DIR}/Build/${TARGET}/Components"
|
||||
export LD_LIBRARY_PATH="\
|
||||
${BUILD_DIR}/Components/DataSources/UDPStreamer:\
|
||||
${BUILD_DIR}/Components/Interfaces/UDPStream:\
|
||||
${MARTe2_DIR}/Build/${TARGET}/Core:\
|
||||
${COMP}/DataSources/LinuxTimer:\
|
||||
${COMP}/DataSources/FileDataSource:\
|
||||
${COMP}/GAMs/IOGAM:\
|
||||
${LD_LIBRARY_PATH:-}"
|
||||
|
||||
MARTE_APP="${MARTe2_DIR}/Build/${TARGET}/App/MARTeApp.ex"
|
||||
STREAMHUB_EX="${BUILD_DIR}/StreamHub/StreamHub.ex"
|
||||
CLIENT="${SCRIPT_DIR}/client/chain-client"
|
||||
PY="python3"
|
||||
|
||||
# ── Build ────────────────────────────────────────────────────────────────────
|
||||
if [ "${SKIP_BUILD}" -eq 0 ]; then
|
||||
echo "── Building components ──"
|
||||
make -C "${REPO_ROOT}/Source/Components/Interfaces/UDPStream" -f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -1
|
||||
make -C "${REPO_ROOT}/Source/Components/DataSources/UDPStreamer" -f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -1
|
||||
make -C "${REPO_ROOT}/Source/Applications/StreamHub" -f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -1
|
||||
fi
|
||||
if [ ! -x "${CLIENT}" ] || [ "${SKIP_BUILD}" -eq 0 ]; then
|
||||
echo "── Building chain-client ──"
|
||||
(cd "${SCRIPT_DIR}/client" && go build -o chain-client .) || { echo "client build failed"; exit 1; }
|
||||
fi
|
||||
[ -x "${MARTE_APP}" ] || { echo "ERROR: MARTeApp.ex not found at ${MARTE_APP}" >&2; exit 1; }
|
||||
[ -x "${STREAMHUB_EX}" ] || { echo "ERROR: StreamHub.ex not found" >&2; exit 1; }
|
||||
|
||||
# ── Validate the matrix, then run ────────────────────────────────────────────
|
||||
${PY} "${SCRIPT_DIR}/stress.py" >/dev/null || { echo "stress matrix invalid"; exit 1; }
|
||||
|
||||
ARGS=(--marte "${MARTE_APP}" --hub "${STREAMHUB_EX}" --client "${CLIENT}"
|
||||
--work "${WORK}" --out "${OUT_DIR}")
|
||||
[ -n "${ONLY}" ] && ARGS+=(--only "${ONLY}")
|
||||
[ -n "${AXIS}" ] && ARGS+=(--axis "${AXIS}")
|
||||
|
||||
${PY} "${SCRIPT_DIR}/stress_run.py" "${ARGS[@]}"
|
||||
RC=$?
|
||||
|
||||
echo ""
|
||||
echo "Done — results in ${OUT_DIR}/stress_results.json, logs in ${OUT_DIR}"
|
||||
exit ${RC}
|
||||
@@ -0,0 +1,589 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
scenarios.py — Declarative scenario matrix for the streaming-chain E2E suite.
|
||||
|
||||
Each scenario describes one full-chain run:
|
||||
MARTe2 App (UDPStreamer) -> StreamHub -> mock client
|
||||
|
||||
The matrix is a *curated covering set*: every configurable option value of the
|
||||
UDPStreamer appears in at least one scenario, plus deliberately chosen
|
||||
high-risk interactions. gen_data.py / gen_cfg.py / client / validate_waveform.py
|
||||
all consume the scenario dict schema documented below.
|
||||
|
||||
Scenario dict schema
|
||||
--------------------
|
||||
{
|
||||
"id": str # unique slug, also the artifact basename
|
||||
"desc": str # human-readable one-liner for the report
|
||||
"network": "unicast" | "multicast"
|
||||
"publishing": "Strict" | "Accumulate" | "Decimate"
|
||||
"ratio": int | None # Decimate only (>=1)
|
||||
"min_refresh_hz": float | None # Accumulate only (>0)
|
||||
"max_payload": int # MaxPayloadSize bytes
|
||||
"ws_port": int # StreamHub WSPort (unique per scenario)
|
||||
"sources": [ # one or more UDPStreamer blocks
|
||||
{
|
||||
"id": str # source/session id (hub Source key)
|
||||
"udp_port": int # UDPStreamer Port (unique per scenario)
|
||||
"data_port": int | None # multicast DATA port
|
||||
"multicast_group": str | None # e.g. "239.0.0.1"
|
||||
"signals": [
|
||||
{
|
||||
"name": str
|
||||
"type": one of TYPE_CODES keys
|
||||
"elements": int # NumberOfElements (1 = scalar)
|
||||
"time_mode": "PacketTime"|"FullArray"|"FirstSample"|"LastSample"
|
||||
"time_signal": str | None # name of the TimeSignal (when needed)
|
||||
"sampling_rate": float | None # Hz, for First/LastSample
|
||||
"quant": "none"|"uint8"|"int8"|"uint16"|"int16"
|
||||
"range_min": float | None # quant only
|
||||
"range_max": float | None # quant only
|
||||
"unit": str | None # e.g. "V", "us", "ns"
|
||||
"formula": "sine"|"ramp"|"counter"|"time_ns"|"time_us"
|
||||
"freq": float | None # sine frequency (Hz over row index)
|
||||
"is_time": bool # True if this signal is a TimeSignal
|
||||
}, ...
|
||||
]
|
||||
}, ...
|
||||
],
|
||||
"oracle": "analytic" | "fed" | "both"
|
||||
"client_checks": subset of ["live","zoom","window","trigger"]
|
||||
"trig_signal": "src:sig" | None # which signal to trigger on
|
||||
"known_issue": str | None # if set, the scenario exercises a
|
||||
# documented, not-yet-fixed chain gap:
|
||||
# a FAIL is reclassified XFAIL (does not
|
||||
# break the baseline) and a PASS becomes
|
||||
# XPASS (the bug is unexpectedly fixed —
|
||||
# time to drop the marker). The string is
|
||||
# the human-readable reason.
|
||||
}
|
||||
"""
|
||||
|
||||
# MARTe2 TypeDescriptor.all codes: all = (type<<2) | (bits<<6),
|
||||
# BasicType SignedInteger=0, UnsignedInteger=1, Float=2 (verified against
|
||||
# MARTe2 L0Types/BasicType.h and L1Portability/TypeDescriptor.h).
|
||||
TYPE_CODES = {
|
||||
"int8": 0x0200,
|
||||
"uint8": 0x0204,
|
||||
"int16": 0x0400,
|
||||
"uint16": 0x0404,
|
||||
"int32": 0x0800,
|
||||
"uint32": 0x0804,
|
||||
"int64": 0x1000,
|
||||
"uint64": 0x1004,
|
||||
"float32": 0x0808,
|
||||
"float64": 0x1008,
|
||||
}
|
||||
|
||||
# numpy dtype per MARTe type (little-endian).
|
||||
NP_DTYPE = {
|
||||
"int8": "<i1", "uint8": "<u1", "int16": "<i2", "uint16": "<u2",
|
||||
"int32": "<i4", "uint32": "<u4", "int64": "<i8", "uint64": "<u8",
|
||||
"float32": "<f4", "float64": "<f8",
|
||||
}
|
||||
|
||||
# Wire constraints shared with the C++ UDPS layer. The UDPStreamer fragments the
|
||||
# serialised packet into chunks of at most MaxPayloadSize *payload* bytes and
|
||||
# prepends a UDPS_HEADER_SIZE-byte header, so the datagram it hands to sendto() is
|
||||
# MaxPayloadSize+UDPS_HEADER_SIZE bytes. That must not exceed the maximum UDP
|
||||
# payload or sendto() fails with EMSGSIZE.
|
||||
UDPS_HEADER_SIZE = 17 # bytes; mirrors Common/UDP/UDPSProtocol.h
|
||||
MAX_UDP_PAYLOAD = 65507 # 65535 - 20 (IP) - 8 (UDP)
|
||||
|
||||
FLOAT_TYPES = {"float32", "float64"}
|
||||
QUANT_TYPES = {"none", "uint8", "int8", "uint16", "int16"}
|
||||
QUANT_LEVELS = {"uint8": 255, "int8": 254, "uint16": 65535, "int16": 65534}
|
||||
TIME_MODES = {"PacketTime", "FullArray", "FirstSample", "LastSample"}
|
||||
|
||||
# Producer buffer geometry — the single source of truth shared with gen_data.py.
|
||||
# The MARTe FileReader loops this finite buffer (EOF=Rewind), so the streamed
|
||||
# signal is only a continuous waveform if the buffer holds an integer number of
|
||||
# periods. The buffer fundamental LOOP_HZ = 1/(NUM_ROWS*ROW_DT) is therefore the
|
||||
# smallest sine frequency that loops seamlessly; every sine freq must be a
|
||||
# positive integer multiple of it or the analytic shape oracle is invalid.
|
||||
NUM_ROWS = 200 # producer cycles written to the FileReader input
|
||||
ROW_DT = 1.0e-3 # seconds per producer cycle (row); 1 kHz producer
|
||||
LOOP_HZ = 1.0 / (NUM_ROWS * ROW_DT) # 5.0 Hz buffer fundamental
|
||||
|
||||
|
||||
def geometry(s):
|
||||
"""Resolve a scenario's producer geometry (per-scenario override or global).
|
||||
|
||||
Returns (row_dt, num_rows, producer_hz, loop_hz). Every data/config/validator
|
||||
consumer reads geometry through here so the override cannot drift apart from
|
||||
the seamless-loop fundamental used by the sine constraint.
|
||||
"""
|
||||
row_dt = s.get("row_dt") or ROW_DT
|
||||
num_rows = s.get("num_rows") or NUM_ROWS
|
||||
producer_hz = s.get("producer_hz") or int(round(1.0 / row_dt))
|
||||
loop_hz = 1.0 / (num_rows * row_dt)
|
||||
return row_dt, num_rows, producer_hz, loop_hz
|
||||
|
||||
|
||||
def _sig(name, type, elements=1, time_mode="PacketTime", time_signal=None,
|
||||
sampling_rate=None, quant="none", range_min=None, range_max=None,
|
||||
unit=None, formula="sine", freq=1.0, is_time=False):
|
||||
return {
|
||||
"name": name, "type": type, "elements": elements,
|
||||
"time_mode": time_mode, "time_signal": time_signal,
|
||||
"sampling_rate": sampling_rate, "quant": quant,
|
||||
"range_min": range_min, "range_max": range_max, "unit": unit,
|
||||
"formula": formula, "freq": freq, "is_time": is_time,
|
||||
}
|
||||
|
||||
|
||||
def validate_scenario(s):
|
||||
"""Return a list of validity error strings (empty == valid)."""
|
||||
errs = []
|
||||
row_dt, num_rows, _producer_hz, loop_hz = geometry(s)
|
||||
if s.get("network") not in ("unicast", "multicast"):
|
||||
errs.append("network must be unicast|multicast")
|
||||
if s.get("publishing") not in ("Strict", "Accumulate", "Decimate"):
|
||||
errs.append("publishing invalid")
|
||||
if s["publishing"] == "Decimate" and not (s.get("ratio") and s["ratio"] >= 1):
|
||||
errs.append("Decimate needs ratio>=1")
|
||||
if s["publishing"] == "Accumulate" and not (s.get("min_refresh_hz") and
|
||||
s["min_refresh_hz"] > 0):
|
||||
errs.append("Accumulate needs min_refresh_hz>0")
|
||||
if not s.get("sources"):
|
||||
errs.append("no sources")
|
||||
max_payload = s.get("max_payload")
|
||||
if max_payload is not None and max_payload + UDPS_HEADER_SIZE > MAX_UDP_PAYLOAD:
|
||||
errs.append(
|
||||
f"max_payload {max_payload} + {UDPS_HEADER_SIZE}B header exceeds the "
|
||||
f"{MAX_UDP_PAYLOAD}B UDP datagram limit (sendto EMSGSIZE); "
|
||||
f"cap at {MAX_UDP_PAYLOAD - UDPS_HEADER_SIZE}")
|
||||
for src in s.get("sources", []):
|
||||
if s["network"] == "multicast":
|
||||
if not src.get("multicast_group") or not src.get("data_port"):
|
||||
errs.append(f"src {src['id']}: multicast needs group+data_port")
|
||||
names = {sig["name"] for sig in src["signals"]}
|
||||
for sig in src["signals"]:
|
||||
if sig["type"] not in TYPE_CODES:
|
||||
errs.append(f"{sig['name']}: bad type {sig['type']}")
|
||||
if sig["quant"] not in QUANT_TYPES:
|
||||
errs.append(f"{sig['name']}: bad quant {sig['quant']}")
|
||||
if sig["quant"] != "none":
|
||||
if sig["type"] not in FLOAT_TYPES:
|
||||
errs.append(f"{sig['name']}: quant only on float types")
|
||||
if sig["range_min"] is None or sig["range_max"] is None:
|
||||
errs.append(f"{sig['name']}: quant needs range_min/max")
|
||||
if sig["time_mode"] not in TIME_MODES:
|
||||
errs.append(f"{sig['name']}: bad time_mode")
|
||||
if sig["time_mode"] != "PacketTime":
|
||||
if not sig["time_signal"] or sig["time_signal"] not in names:
|
||||
errs.append(f"{sig['name']}: time_mode needs valid time_signal")
|
||||
if sig["time_mode"] == "FullArray":
|
||||
ts = next((x for x in src["signals"]
|
||||
if x["name"] == sig["time_signal"]), None)
|
||||
if ts and ts["elements"] != sig["elements"]:
|
||||
errs.append(f"{sig['name']}: FullArray time_signal length mismatch")
|
||||
if sig["time_mode"] in ("FirstSample", "LastSample"):
|
||||
if not sig["sampling_rate"] or sig["sampling_rate"] <= 0:
|
||||
errs.append(f"{sig['name']}: First/LastSample needs sampling_rate>0")
|
||||
else:
|
||||
# The per-packet sample window must fit inside one producer
|
||||
# cycle, or successive packets' expanded timestamps overlap
|
||||
# and the hub's binary-search ring (which assumes a sorted
|
||||
# time axis) is corrupted — the s17/s18 failure class.
|
||||
window = (sig["elements"] - 1) / sig["sampling_rate"]
|
||||
if window > row_dt + 1e-12:
|
||||
errs.append(
|
||||
f"{sig['name']}: {sig['time_mode']} window "
|
||||
f"{window * 1e3:.4f} ms > row period {row_dt * 1e3:.4f} ms "
|
||||
f"(non-monotonic ring); raise producer rate or "
|
||||
f"sampling_rate, or lower elements")
|
||||
if sig["formula"] == "sine" and sig.get("freq"):
|
||||
ratio = sig["freq"] / loop_hz
|
||||
if abs(ratio - round(ratio)) > 1e-9 or round(ratio) < 1:
|
||||
errs.append(f"{sig['name']}: sine freq {sig['freq']} must be a "
|
||||
f"positive multiple of LOOP_HZ={loop_hz} (seamless loop)")
|
||||
return errs
|
||||
|
||||
|
||||
# ── Curated covering matrix ───────────────────────────────────────────────────
|
||||
# The first three scenarios are referenced positionally by tests_py.py
|
||||
# (SCENARIOS[0..2]) and are kept verbatim. The remainder is built with the helpers
|
||||
# below, which auto-allocate unique WS/UDP/DATA ports so the matrix can grow
|
||||
# without manual bookkeeping. Coverage goal: every UDPStreamer option *value*
|
||||
# (all 10 types, scalar+array shapes, all four TimeModes, all five quant kinds,
|
||||
# the three publishing modes, unicast+multicast, fragmentation via small payload,
|
||||
# multi-source) appears at least once, plus deliberately chosen high-risk
|
||||
# interactions (decimate+quant+array, accumulate+fullarray, multicast+decimate,
|
||||
# fragmentation+decimate). Non-sine formulas (counter/ramp) are preferred for
|
||||
# pure type/shape coverage because only the fidelity oracle gates them; sine is
|
||||
# used where the shape metric should be tracked. MCAST_GROUP must have a route on
|
||||
# the test host or those scenarios report SKIP (the orchestrator probes it).
|
||||
import itertools
|
||||
|
||||
MCAST_GROUP = "239.0.7.7"
|
||||
_ws = itertools.count(8104)
|
||||
_udp = itertools.count(44616, 2)
|
||||
_data = itertools.count(45616, 2)
|
||||
|
||||
|
||||
def _src(sid, signals, multicast=False):
|
||||
return {
|
||||
"id": sid, "udp_port": next(_udp),
|
||||
"data_port": next(_data) if multicast else None,
|
||||
"multicast_group": MCAST_GROUP if multicast else None,
|
||||
"signals": signals,
|
||||
}
|
||||
|
||||
|
||||
def mk(sid, desc, sources, network="unicast", publishing="Strict",
|
||||
ratio=None, min_refresh_hz=None, max_payload=1400,
|
||||
oracle="analytic", checks=("live", "zoom"), trig=None,
|
||||
known_issue=None, row_dt=None, num_rows=None, producer_hz=None):
|
||||
# row_dt / num_rows / producer_hz override the global producer geometry for
|
||||
# this scenario only (default None == use the suite-wide NUM_ROWS / ROW_DT /
|
||||
# 1 kHz). They are kept together so the per-cycle wall gap (1/producer_hz),
|
||||
# the encoded time-step (row_dt) and the seamless-loop fundamental
|
||||
# (1/(num_rows*row_dt)) stay mutually consistent.
|
||||
return {
|
||||
"id": sid, "desc": desc, "network": network, "publishing": publishing,
|
||||
"ratio": ratio, "min_refresh_hz": min_refresh_hz,
|
||||
"max_payload": max_payload, "ws_port": next(_ws),
|
||||
"sources": sources, "oracle": oracle,
|
||||
"client_checks": list(checks), "trig_signal": trig,
|
||||
"known_issue": known_issue,
|
||||
"row_dt": row_dt, "num_rows": num_rows, "producer_hz": producer_hz,
|
||||
}
|
||||
|
||||
|
||||
_STARTERS = [
|
||||
{
|
||||
"id": "s01_scalar_uint32",
|
||||
"desc": "Single uint32 scalar counter, Strict unicast (type fidelity)",
|
||||
"network": "unicast", "publishing": "Strict",
|
||||
"ratio": None, "min_refresh_hz": None, "max_payload": 1400,
|
||||
"ws_port": 8101,
|
||||
"sources": [{
|
||||
"id": "src", "udp_port": 44610, "data_port": None,
|
||||
"multicast_group": None,
|
||||
"signals": [
|
||||
_sig("Counter", "uint32", 1, formula="counter"),
|
||||
_sig("Sine", "float32", 1, formula="sine", freq=5.0, unit="V"),
|
||||
],
|
||||
}],
|
||||
"oracle": "analytic",
|
||||
"client_checks": ["live", "zoom", "window", "trigger"],
|
||||
"trig_signal": "src:Sine",
|
||||
},
|
||||
{
|
||||
"id": "s02_array_float32_fullarray",
|
||||
"desc": "100-elem float32 array, FullArray time mode, uint64 ns time array",
|
||||
"network": "unicast", "publishing": "Strict",
|
||||
"ratio": None, "min_refresh_hz": None,
|
||||
"max_payload": MAX_UDP_PAYLOAD - UDPS_HEADER_SIZE,
|
||||
"ws_port": 8102,
|
||||
"sources": [{
|
||||
"id": "src", "udp_port": 44612, "data_port": None,
|
||||
"multicast_group": None,
|
||||
"signals": [
|
||||
_sig("TimeArr", "uint64", 100, unit="ns",
|
||||
formula="time_ns", is_time=True),
|
||||
_sig("Wave", "float32", 100, time_mode="FullArray",
|
||||
time_signal="TimeArr", formula="sine", freq=5.0, unit="V"),
|
||||
],
|
||||
}],
|
||||
"oracle": "both",
|
||||
"client_checks": ["live", "zoom"],
|
||||
"trig_signal": None,
|
||||
},
|
||||
{
|
||||
"id": "s03_quant_uint16",
|
||||
"desc": "float32 scalar quantised to uint16 over [-5,5], Strict unicast",
|
||||
"network": "unicast", "publishing": "Strict",
|
||||
"ratio": None, "min_refresh_hz": None, "max_payload": 1400,
|
||||
"ws_port": 8103,
|
||||
"sources": [{
|
||||
"id": "src", "udp_port": 44614, "data_port": None,
|
||||
"multicast_group": None,
|
||||
"signals": [
|
||||
_sig("Sine", "float32", 1, quant="uint16",
|
||||
range_min=-5.0, range_max=5.0, formula="sine",
|
||||
freq=10.0, unit="V"),
|
||||
],
|
||||
}],
|
||||
"oracle": "analytic",
|
||||
"client_checks": ["live", "zoom"],
|
||||
"trig_signal": "src:Sine",
|
||||
},
|
||||
]
|
||||
|
||||
# ── Type fidelity: one scalar per remaining MARTe type (fidelity-only) ────────
|
||||
_TYPES = [
|
||||
mk("s04_int8_scalar", "int8 scalar counter, type fidelity",
|
||||
[_src("src", [_sig("Cnt", "int8", 1, formula="counter")])]),
|
||||
mk("s05_uint8_scalar", "uint8 scalar counter, type fidelity",
|
||||
[_src("src", [_sig("Cnt", "uint8", 1, formula="counter")])]),
|
||||
mk("s06_int16_scalar", "int16 scalar ramp, type fidelity",
|
||||
[_src("src", [_sig("Ramp", "int16", 1, formula="ramp")])]),
|
||||
mk("s07_uint16_scalar", "uint16 scalar ramp, type fidelity",
|
||||
[_src("src", [_sig("Ramp", "uint16", 1, formula="ramp")])]),
|
||||
mk("s08_int32_scalar", "int32 scalar counter, type fidelity",
|
||||
[_src("src", [_sig("Cnt", "int32", 1, formula="counter")])]),
|
||||
mk("s09_int64_scalar", "int64 scalar counter, type fidelity",
|
||||
[_src("src", [_sig("Cnt", "int64", 1, formula="counter")])]),
|
||||
mk("s10_uint64_scalar", "uint64 scalar counter, type fidelity",
|
||||
[_src("src", [_sig("Cnt", "uint64", 1, formula="counter")])]),
|
||||
mk("s11_float64_scalar", "float64 scalar sine 5 Hz (double-precision path)",
|
||||
[_src("src", [_sig("Sine", "float64", 1, formula="sine", freq=5.0,
|
||||
unit="V")])],
|
||||
checks=("live", "zoom", "trigger"), trig="src:Sine"),
|
||||
]
|
||||
|
||||
# ── Array shapes (NumberOfElements) ──────────────────────────────────────────
|
||||
_ARRAYS = [
|
||||
mk("s12_f32_arr8", "float32 8-elem array sine 5 Hz",
|
||||
[_src("src", [_sig("Wave", "float32", 8, formula="sine", freq=5.0)])]),
|
||||
mk("s13_f32_arr32", "float32 32-elem array sine 10 Hz",
|
||||
[_src("src", [_sig("Wave", "float32", 32, formula="sine", freq=10.0)])]),
|
||||
mk("s14_f64_arr64", "float64 64-elem array ramp",
|
||||
[_src("src", [_sig("Ramp", "float64", 64, formula="ramp")])]),
|
||||
mk("s15_i16_arr16", "int16 16-elem array counter",
|
||||
[_src("src", [_sig("Cnt", "int16", 16, formula="counter")])]),
|
||||
mk("s16_f32_arr256", "float32 256-elem array sine 5 Hz (large frame)",
|
||||
[_src("src", [_sig("Wave", "float32", 256, formula="sine", freq=5.0)])],
|
||||
max_payload=MAX_UDP_PAYLOAD - UDPS_HEADER_SIZE),
|
||||
mk("s39_uint8_arr32", "uint8 32-elem array counter (wrap fidelity)",
|
||||
[_src("src", [_sig("Cnt", "uint8", 32, formula="counter")])]),
|
||||
mk("s40_int8_arr16", "int8 16-elem array counter (wrap fidelity)",
|
||||
[_src("src", [_sig("Cnt", "int8", 16, formula="counter")])]),
|
||||
]
|
||||
|
||||
# ── Time modes (each non-PacketTime mode needs a TimeSignal in the source) ────
|
||||
_TIMEMODES = [
|
||||
# sampling_rate = elements/ROW_DT (8/0.001 = 8000) so the per-cycle window
|
||||
# of 8 samples fills exactly one 1 kHz producer cycle. A smaller rate makes
|
||||
# each cycle's window wider than the inter-cycle gap, so successive windows
|
||||
# overlap and the published timestamps stop being monotonic — which corrupts
|
||||
# the hub's binary-search range/zoom queries (they assume a sorted ring).
|
||||
mk("s17_lastsample", "float32 8-elem LastSample, uint64 ns scalar anchor",
|
||||
[_src("src", [
|
||||
_sig("Tns", "uint64", 1, unit="ns", formula="time_ns", is_time=True),
|
||||
_sig("Data", "float32", 8, time_mode="LastSample",
|
||||
time_signal="Tns", sampling_rate=8000.0, formula="counter"),
|
||||
])]),
|
||||
mk("s18_firstsample", "float32 8-elem FirstSample, uint32 us scalar anchor",
|
||||
[_src("src", [
|
||||
_sig("Tus", "uint32", 1, unit="us", formula="time_us", is_time=True),
|
||||
_sig("Data", "float32", 8, time_mode="FirstSample",
|
||||
time_signal="Tus", sampling_rate=8000.0, formula="counter"),
|
||||
])]),
|
||||
mk("s19_fullarray_f64", "float64 50-elem FullArray sine 5 Hz, uint64 ns time",
|
||||
[_src("src", [
|
||||
_sig("TimeArr", "uint64", 50, unit="ns", formula="time_ns",
|
||||
is_time=True),
|
||||
_sig("Wave", "float64", 50, time_mode="FullArray",
|
||||
time_signal="TimeArr", formula="sine", freq=5.0, unit="V"),
|
||||
])],
|
||||
oracle="both"),
|
||||
mk("s43_fullarray_quant", "float32 16-elem FullArray quant uint16 sine 5 Hz",
|
||||
[_src("src", [
|
||||
_sig("TimeArr", "uint64", 16, unit="ns", formula="time_ns",
|
||||
is_time=True),
|
||||
_sig("Wave", "float32", 16, time_mode="FullArray",
|
||||
time_signal="TimeArr", quant="uint16", range_min=-1.0,
|
||||
range_max=1.0, formula="sine", freq=5.0, unit="V"),
|
||||
])]),
|
||||
]
|
||||
|
||||
# ── Quantization (each QuantizedType, on float signals) ──────────────────────
|
||||
_QUANT = [
|
||||
mk("s20_quant_uint8", "float32 scalar quant uint8 [-1,1] sine 5 Hz",
|
||||
[_src("src", [_sig("Sine", "float32", 1, quant="uint8", range_min=-1.0,
|
||||
range_max=1.0, formula="sine", freq=5.0)])]),
|
||||
mk("s21_quant_int8", "float32 scalar quant int8 [-10,10] sine 5 Hz",
|
||||
[_src("src", [_sig("Sine", "float32", 1, quant="int8", range_min=-10.0,
|
||||
range_max=10.0, formula="sine", freq=5.0)])]),
|
||||
mk("s22_quant_int16", "float32 scalar quant int16 [-100,100] ramp",
|
||||
[_src("src", [_sig("Ramp", "float32", 1, quant="int16", range_min=-100.0,
|
||||
range_max=100.0, formula="ramp")])]),
|
||||
mk("s23_quant_f64_arr", "float64 16-elem quant uint16 [-2,2] sine 5 Hz",
|
||||
[_src("src", [_sig("Wave", "float64", 16, quant="uint16", range_min=-2.0,
|
||||
range_max=2.0, formula="sine", freq=5.0)])]),
|
||||
]
|
||||
|
||||
# ── Publishing modes (Strict covered; Accumulate + Decimate here) ────────────
|
||||
_PUBLISH = [
|
||||
mk("s24_accumulate", "float32 scalar sine 5 Hz, Accumulate @50 Hz refresh",
|
||||
[_src("src", [_sig("Sine", "float32", 1, formula="sine", freq=5.0)])],
|
||||
publishing="Accumulate", min_refresh_hz=50.0),
|
||||
mk("s25_decimate4", "float32 scalar sine 5 Hz, Decimate ratio 4",
|
||||
[_src("src", [_sig("Sine", "float32", 1, formula="sine", freq=5.0)])],
|
||||
publishing="Decimate", ratio=4),
|
||||
mk("s26_decimate10_arr", "float32 8-elem counter, Decimate ratio 10",
|
||||
[_src("src", [_sig("Cnt", "float32", 8, formula="counter")])],
|
||||
publishing="Decimate", ratio=10),
|
||||
mk("s46_accumulate_arr",
|
||||
"Accumulate @200 Hz: accumulated scalar sine + 16-elem array passenger",
|
||||
[_src("src", [_sig("Sine", "float32", 1, formula="sine", freq=5.0),
|
||||
_sig("Wave", "float32", 16, formula="sine", freq=5.0)])],
|
||||
publishing="Accumulate", min_refresh_hz=200.0),
|
||||
mk("s45_decimate_multisig", "Decimate ratio 2 over a 2-signal source",
|
||||
[_src("src", [_sig("Sine", "float32", 1, formula="sine", freq=5.0),
|
||||
_sig("Cnt", "uint32", 1, formula="counter")])],
|
||||
publishing="Decimate", ratio=2),
|
||||
]
|
||||
|
||||
# ── Payload size / fragmentation (small MaxPayloadSize vs large array) ───────
|
||||
_PAYLOAD = [
|
||||
mk("s27_frag_f64_128", "float64 128-elem ramp, MaxPayload 512 (fragmented)",
|
||||
[_src("src", [_sig("Ramp", "float64", 128, formula="ramp")])],
|
||||
max_payload=512),
|
||||
mk("s28_frag_f32_100", "float32 100-elem sine 5 Hz, MaxPayload 256 (fragmented)",
|
||||
[_src("src", [_sig("Wave", "float32", 100, formula="sine", freq=5.0)])],
|
||||
max_payload=256),
|
||||
mk("s48_f64_arr_big_payload", "float64 100-elem ramp, MaxPayload 65490 (single frame)",
|
||||
[_src("src", [_sig("Ramp", "float64", 100, formula="ramp")])],
|
||||
max_payload=MAX_UDP_PAYLOAD - UDPS_HEADER_SIZE),
|
||||
]
|
||||
|
||||
# ── Multicast ────────────────────────────────────────────────────────────────
|
||||
_MCAST = [
|
||||
mk("s29_mcast_scalar", "multicast float32 scalar sine 5 Hz",
|
||||
[_src("src", [_sig("Sine", "float32", 1, formula="sine", freq=5.0)],
|
||||
multicast=True)],
|
||||
network="multicast"),
|
||||
mk("s30_mcast_arr_fullarray", "multicast float32 32-elem FullArray sine 5 Hz",
|
||||
[_src("src", [
|
||||
_sig("TimeArr", "uint64", 32, unit="ns", formula="time_ns",
|
||||
is_time=True),
|
||||
_sig("Wave", "float32", 32, time_mode="FullArray",
|
||||
time_signal="TimeArr", formula="sine", freq=5.0),
|
||||
], multicast=True)],
|
||||
network="multicast"),
|
||||
mk("s47_mcast_multisrc", "multicast, two sources (scalar each)",
|
||||
[_src("a", [_sig("Sine", "float32", 1, formula="sine", freq=5.0)],
|
||||
multicast=True),
|
||||
_src("b", [_sig("Cnt", "uint32", 1, formula="counter")],
|
||||
multicast=True)],
|
||||
network="multicast"),
|
||||
]
|
||||
|
||||
# ── Multiple sources (independent UDPStreamer blocks, one hub) ───────────────
|
||||
_MULTISRC = [
|
||||
mk("s31_two_src", "two unicast sources: float32 sine + uint32 counter",
|
||||
[_src("a", [_sig("Sine", "float32", 1, formula="sine", freq=5.0)]),
|
||||
_src("b", [_sig("Cnt", "uint32", 1, formula="counter")])]),
|
||||
mk("s32_three_src", "three unicast sources: int16 ramp / float64 sine / uint8 counter",
|
||||
[_src("a", [_sig("Ramp", "int16", 1, formula="ramp")]),
|
||||
_src("b", [_sig("Sine", "float64", 1, formula="sine", freq=5.0)]),
|
||||
_src("c", [_sig("Cnt", "uint8", 1, formula="counter")])]),
|
||||
]
|
||||
|
||||
# ── High-risk interactions ───────────────────────────────────────────────────
|
||||
_INTERACT = [
|
||||
mk("s33_dec_arr_quant", "Decimate 2 + 16-elem quant uint16 sine 5 Hz",
|
||||
[_src("src", [_sig("Wave", "float32", 16, quant="uint16", range_min=-1.0,
|
||||
range_max=1.0, formula="sine", freq=5.0)])],
|
||||
publishing="Decimate", ratio=2),
|
||||
mk("s34_acc_fullarray",
|
||||
"Accumulate @100 Hz: accumulated scalar + 32-elem FullArray sine passenger",
|
||||
[_src("src", [
|
||||
_sig("Sine", "float32", 1, formula="sine", freq=5.0),
|
||||
_sig("TimeArr", "uint64", 32, unit="ns", formula="time_ns",
|
||||
is_time=True),
|
||||
_sig("Wave", "float32", 32, time_mode="FullArray",
|
||||
time_signal="TimeArr", formula="sine", freq=5.0),
|
||||
])],
|
||||
publishing="Accumulate", min_refresh_hz=100.0),
|
||||
mk("s35_mcast_decimate", "multicast + Decimate ratio 5, float32 scalar sine 5 Hz",
|
||||
[_src("src", [_sig("Sine", "float32", 1, formula="sine", freq=5.0)],
|
||||
multicast=True)],
|
||||
network="multicast", publishing="Decimate", ratio=5),
|
||||
mk("s36_big_frag_dec", "float64 64-elem ramp, MaxPayload 256 + Decimate 4",
|
||||
[_src("src", [_sig("Ramp", "float64", 64, formula="ramp")])],
|
||||
max_payload=256, publishing="Decimate", ratio=4),
|
||||
mk("s49_mixed_quant_raw", "one source: quant uint8 sine + raw float32 sine",
|
||||
[_src("src", [
|
||||
_sig("Q", "float32", 1, quant="uint8", range_min=-1.0, range_max=1.0,
|
||||
formula="sine", freq=5.0),
|
||||
_sig("Raw", "float32", 1, formula="sine", freq=5.0),
|
||||
])]),
|
||||
]
|
||||
|
||||
# ── Trigger + client-check coverage (rising/falling/both swept by the client) ─
|
||||
_TRIGGER = [
|
||||
mk("s37_trig_ramp_i32", "trigger on int32 ramp scalar",
|
||||
[_src("src", [_sig("Ramp", "int32", 1, formula="ramp")])],
|
||||
checks=("live", "trigger"), trig="src:Ramp"),
|
||||
mk("s38_trig_f64_sine", "trigger on float64 sine 5 Hz scalar",
|
||||
[_src("src", [_sig("Sine", "float64", 1, formula="sine", freq=5.0)])],
|
||||
checks=("live", "zoom", "trigger"), trig="src:Sine"),
|
||||
mk("s44_window_check", "float32 sine 5 Hz scalar, window time-range check",
|
||||
[_src("src", [_sig("Sine", "float32", 1, formula="sine", freq=5.0)])],
|
||||
checks=("live", "zoom", "window")),
|
||||
mk("s50_trig_quant", "trigger on quantised uint16 sine 10 Hz",
|
||||
[_src("src", [_sig("Sine", "float32", 1, quant="uint16", range_min=-5.0,
|
||||
range_max=5.0, formula="sine", freq=10.0)])],
|
||||
checks=("live", "trigger"), trig="src:Sine"),
|
||||
]
|
||||
|
||||
# ── Misc scalar coverage (unit annotation, counter on float) ─────────────────
|
||||
_MISC = [
|
||||
mk("s41_f32_unit", "float32 scalar ramp with Unit=V",
|
||||
[_src("src", [_sig("Ramp", "float32", 1, formula="ramp", unit="V")])]),
|
||||
mk("s42_f64_counter", "float64 scalar counter (large integer values)",
|
||||
[_src("src", [_sig("Cnt", "float64", 1, formula="counter")])]),
|
||||
]
|
||||
|
||||
# ── High sample-rate / high-throughput stress ────────────────────────────────
|
||||
# Eight signals, each a 1000-element float32 array carrying 1 ms worth of 1 MSps
|
||||
# data, published once per 1 kHz producer cycle. Aggregate wire rate is
|
||||
# 8 x 1 MSps x 4 B = 32 MB/s. A single scalar uint64 ns anchor (FirstSample) is
|
||||
# shared by all eight data signals; the per-packet sample window
|
||||
# (1000 / 1 MHz = 1 ms) equals the 1 kHz inter-packet gap so the expanded
|
||||
# per-sample timestamps stay monotonic for the hub's binary-search ring. Ramp
|
||||
# formula keeps the gate fidelity-only (robust to the UDP loss expected at rate).
|
||||
#
|
||||
# Packet sizing — DATA payload is 8 (ns anchor) + 8 x 1000 x 4 = 32008 B, which
|
||||
# is deliberately kept under the 64 KB single-datagram ceiling. That ceiling is
|
||||
# the UDPSClient reassembly buffer (UDPSReassemblySlot::payload[65535]) plus its
|
||||
# offset overflow guard: a packet whose payload exceeds ~64 KB can never be
|
||||
# reassembled (fragment 1 lands past the buffer and is dropped), so high-rate
|
||||
# streaming must use single-datagram packets. max_payload is set to the maximum
|
||||
# valid value (UDP limit - 17 B header) so the 32 KB packet is sent unfragmented.
|
||||
_HIGHRATE = [
|
||||
mk("s51_8x1msps_100hz",
|
||||
"8x float32 10k-elem arrays @1 MSps, FirstSample, 100 Hz packets (~32 MB/s, "
|
||||
"320 KB/cycle fragmented)",
|
||||
[_src("src", [
|
||||
_sig("Tns", "uint64", 1, unit="ns", formula="time_ns", is_time=True),
|
||||
] + [
|
||||
_sig(f"S{i}", "float32", 10000, time_mode="FirstSample",
|
||||
time_signal="Tns", sampling_rate=1.0e6, formula="ramp")
|
||||
for i in range(8)
|
||||
])],
|
||||
max_payload=40000, row_dt=0.01, num_rows=50, producer_hz=100,
|
||||
checks=("live", "zoom")),
|
||||
]
|
||||
|
||||
SCENARIOS = (_STARTERS + _TYPES + _ARRAYS + _TIMEMODES + _QUANT + _PUBLISH +
|
||||
_PAYLOAD + _MCAST + _MULTISRC + _INTERACT + _TRIGGER + _MISC +
|
||||
_HIGHRATE)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
ok = True
|
||||
seen_ids, seen_ws, seen_udp = set(), set(), set()
|
||||
for s in SCENARIOS:
|
||||
errs = validate_scenario(s)
|
||||
# uniqueness invariants the orchestrator relies on
|
||||
if s["id"] in seen_ids:
|
||||
errs.append("duplicate id")
|
||||
seen_ids.add(s["id"])
|
||||
if s["ws_port"] in seen_ws:
|
||||
errs.append(f"duplicate ws_port {s['ws_port']}")
|
||||
seen_ws.add(s["ws_port"])
|
||||
for src in s["sources"]:
|
||||
if src["udp_port"] in seen_udp:
|
||||
errs.append(f"duplicate udp_port {src['udp_port']}")
|
||||
seen_udp.add(src["udp_port"])
|
||||
print(f"{s['id']:32s} {'OK' if not errs else errs}")
|
||||
ok = ok and not errs
|
||||
print(f"\n{len(SCENARIOS)} scenarios, {'ALL VALID' if ok else 'INVALID PRESENT'}")
|
||||
sys.exit(0 if ok else 1)
|
||||
@@ -0,0 +1,237 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
stress.py — Declarative stress matrix for the streaming chain.
|
||||
|
||||
Where scenarios.py exercises *correctness* (every option value, exact waveform
|
||||
fidelity), this module exercises *capacity*: it pushes one axis of load at a time
|
||||
and records how the two server processes behave. The two components under stress:
|
||||
|
||||
* UDPStreamer (datasource) — serialises + sends UDPS packets each RT cycle.
|
||||
Stress axes: signal size (bytes/packet), signal count, subscriber count
|
||||
(UDPStreamer fans a unicast source out to up to 16 clients).
|
||||
* StreamHub (hub) — ring storage, LTTB decimation, WS fan-out, zoom.
|
||||
Stress axes: signal size, source count (independent UDPStreamer feeds),
|
||||
WS client count, and client request (zoom) rate.
|
||||
|
||||
What is measured (not waveform fidelity — at these rates UDP loss is expected and
|
||||
the hub decimates to PushRate anyway, so end-to-end sample loss is not the gate):
|
||||
|
||||
* survival — neither server crashed/hung for the whole run (hard gate).
|
||||
* liveness — every WS client kept receiving monotonic, wall-clock-stamped
|
||||
pushes under the load (hard gate).
|
||||
* cpu / peakRSS — of marte and hub, captured by proc_perf.py (scaling curves;
|
||||
soft gate against generous ceilings).
|
||||
* zoom latency — p50/p95 round-trip of zoom queries issued under load (soft
|
||||
gate; the headline responsiveness metric for the hub).
|
||||
|
||||
Stress-case dict schema (a superset of the scenarios.py scenario dict, so the
|
||||
gen_data.py / gen_cfg.py generators consume it unchanged):
|
||||
|
||||
{ ...all scenario keys (id, network, publishing, max_payload, ws_port,
|
||||
sources, oracle, client_checks, trig_signal, row_dt/num_rows/...),
|
||||
"shape": "hub" | "ds_fanout",
|
||||
"stress": {
|
||||
"axis": str, # which load axis this case varies (for the report)
|
||||
"level": number, # the axis value (x for the scaling curve)
|
||||
"clients": int, # parallel WS clients (hub shape)
|
||||
"hubs": int, # parallel subscriber hubs (ds_fanout shape)
|
||||
"reqrate": float, # zoom requests/sec/client (0 = one-shot checks)
|
||||
"dur": float, # live-load duration (s)
|
||||
"gate": { "min_frames": int, "max_marte_rss_mb": float,
|
||||
"max_hub_rss_mb": float, "max_zoom_p95_ms": float },
|
||||
} }
|
||||
|
||||
The matrix keeps every datagram a single UDP fragment (payload < 64 KB): the
|
||||
UDPSClient reassembly buffer caps a deliverable packet at ~64 KB, so sustained
|
||||
high-rate streaming must stay below it (see scenarios.py s51). Cases therefore
|
||||
sweep *count* and *rate*, not oversized single packets.
|
||||
"""
|
||||
import itertools
|
||||
|
||||
import scenarios as S
|
||||
|
||||
# Dedicated port ranges, disjoint from scenarios.py (ws 8101-8151, udp 44610-).
|
||||
_ws = itertools.count(8300)
|
||||
_udp = itertools.count(45000, 2)
|
||||
_data = itertools.count(46000, 2)
|
||||
|
||||
# Producer geometry knobs shared across the matrix. 1 kHz producer (default),
|
||||
# float32 arrays sized so one cycle's array is a single sub-64 KB datagram.
|
||||
F32 = "float32"
|
||||
|
||||
|
||||
def _f32_arr(name, elements, sampling_rate=1.0e6):
|
||||
"""A float32 array signal timestamped FirstSample off the shared ns anchor.
|
||||
|
||||
sampling_rate defaults to elements*producer_hz so the per-cycle window equals
|
||||
one 1 kHz producer cycle (keeps the hub ring's time axis monotonic, the same
|
||||
constraint scenarios.py enforces for First/LastSample)."""
|
||||
return S._sig(name, F32, elements, time_mode="FirstSample",
|
||||
time_signal="Tns", sampling_rate=sampling_rate, formula="ramp")
|
||||
|
||||
|
||||
def _source(sid, n_signals, elements, multicast=False):
|
||||
"""One UDPStreamer source: a uint64 ns anchor + n_signals float32 arrays.
|
||||
|
||||
sampling_rate = elements / row_dt(1 ms) = elements * 1000 keeps each array's
|
||||
1 ms window aligned to the 1 kHz cycle regardless of `elements`."""
|
||||
rate = elements * 1000.0
|
||||
sigs = [S._sig("Tns", "uint64", 1, unit="ns", formula="time_ns",
|
||||
is_time=True)]
|
||||
sigs += [_f32_arr(f"S{i}", elements, rate) for i in range(n_signals)]
|
||||
return {
|
||||
"id": sid, "udp_port": next(_udp),
|
||||
"data_port": next(_data) if multicast else None,
|
||||
"multicast_group": S.MCAST_GROUP if multicast else None,
|
||||
"signals": sigs,
|
||||
}
|
||||
|
||||
|
||||
def _packet_bytes(n_signals, elements):
|
||||
"""Approx DATA payload bytes/packet: ns anchor + n_signals*elements*4."""
|
||||
return 8 + n_signals * elements * 4
|
||||
|
||||
|
||||
def mk_stress(sid, axis, level, sources, shape="hub", clients=1, hubs=1,
|
||||
reqrate=0.0, dur=6.0, network="unicast", publishing="Strict",
|
||||
ratio=None, min_refresh_hz=None, gate=None):
|
||||
case = {
|
||||
"id": sid, "desc": f"stress {axis}={level}",
|
||||
"network": network, "publishing": publishing,
|
||||
"ratio": ratio, "min_refresh_hz": min_refresh_hz,
|
||||
"max_payload": S.MAX_UDP_PAYLOAD - S.UDPS_HEADER_SIZE,
|
||||
"ws_port": next(_ws), "sources": sources,
|
||||
"oracle": "analytic", "client_checks": ["live", "zoom"],
|
||||
"trig_signal": None, "known_issue": None,
|
||||
"row_dt": None, "num_rows": None, "producer_hz": None,
|
||||
"shape": shape,
|
||||
"stress": {
|
||||
"axis": axis, "level": level, "clients": clients, "hubs": hubs,
|
||||
"reqrate": reqrate, "dur": dur,
|
||||
"gate": gate or {},
|
||||
},
|
||||
}
|
||||
return case
|
||||
|
||||
|
||||
# Default gate: at least a handful of frames per client, generous RSS ceilings,
|
||||
# and a 1 s p95 zoom round-trip cap. Stress cases tune these per axis.
|
||||
def _gate(min_frames=5, marte_rss=512.0, hub_rss=1024.0, zoom_p95=1000.0):
|
||||
return {"min_frames": min_frames, "max_marte_rss_mb": marte_rss,
|
||||
"max_hub_rss_mb": hub_rss, "max_zoom_p95_ms": zoom_p95}
|
||||
|
||||
|
||||
# ── UDPStreamer (datasource) stress ───────────────────────────────────────────
|
||||
# Single source + single hub; the load lands on the UDPStreamer serialise/send
|
||||
# path and is read back through one WS client (proc_perf measures marte).
|
||||
|
||||
# size: one float32 array, growing element count → bigger single-datagram packet.
|
||||
_DS_SIZE = [
|
||||
mk_stress(f"ds_size_{e}", "ds_signal_elements", e,
|
||||
[_source("src", 1, e)], gate=_gate())
|
||||
for e in (1000, 4000, 8000, 15000) # 4 KB → 60 KB packets (sub-64 KB cap)
|
||||
]
|
||||
|
||||
# count: many 1000-element float32 arrays in one source → wider packet + more
|
||||
# per-cycle serialise work.
|
||||
_DS_COUNT = [
|
||||
mk_stress(f"ds_count_{n}", "ds_signal_count", n,
|
||||
[_source("src", n, 1000)], gate=_gate())
|
||||
for n in (1, 4, 8, 15) # 15*4 KB ≈ 60 KB packet (sub-64 KB cap)
|
||||
]
|
||||
|
||||
# clients (fan-out): one source, M subscriber hubs (ds_fanout shape). The
|
||||
# UDPStreamer copies every packet to each unicast client (max 16); CPU should
|
||||
# scale with M. Each hub gets its own WS port + one client.
|
||||
_DS_CLIENTS = [
|
||||
mk_stress(f"ds_clients_{m}", "ds_subscriber_hubs", m,
|
||||
[_source("src", 4, 2000)], shape="ds_fanout", hubs=m,
|
||||
gate=_gate(hub_rss=1024.0))
|
||||
for m in (1, 2, 4, 8)
|
||||
]
|
||||
|
||||
# ── StreamHub (hub) stress ────────────────────────────────────────────────────
|
||||
# All measured against a single hub; proc_perf measures the hub process.
|
||||
|
||||
# size: hub ring/decimation cost vs per-signal width.
|
||||
_HUB_SIZE = [
|
||||
mk_stress(f"hub_size_{e}", "hub_signal_elements", e,
|
||||
[_source("src", 1, e)], gate=_gate())
|
||||
for e in (1000, 4000, 8000, 15000)
|
||||
]
|
||||
|
||||
# sources: N independent UDPStreamer feeds into one hub (each its own udp_port).
|
||||
_HUB_SOURCES = [
|
||||
mk_stress(f"hub_sources_{n}", "hub_source_count", n,
|
||||
[_source(f"s{i}", 2, 1000) for i in range(n)],
|
||||
gate=_gate(marte_rss=1024.0))
|
||||
for n in (1, 2, 4, 8)
|
||||
]
|
||||
|
||||
# clients: one source, C parallel WS clients all recording live + zooming.
|
||||
_HUB_CLIENTS = [
|
||||
mk_stress(f"hub_clients_{c}", "hub_ws_clients", c,
|
||||
[_source("src", 2, 2000)], clients=c, gate=_gate())
|
||||
for c in (1, 4, 8, 16)
|
||||
]
|
||||
|
||||
# request rate: one source, a few clients each issuing zoom queries at a sustained
|
||||
# rate; the headline gate is zoom p95 latency under that query load.
|
||||
_HUB_REQRATE = [
|
||||
mk_stress(f"hub_reqrate_{r}", "hub_zoom_reqrate_hz", r,
|
||||
[_source("src", 2, 2000)], clients=4, reqrate=r, dur=8.0,
|
||||
gate=_gate(zoom_p95=1500.0))
|
||||
for r in (5, 20, 50)
|
||||
]
|
||||
|
||||
STRESS_CASES = (_DS_SIZE + _DS_COUNT + _DS_CLIENTS +
|
||||
_HUB_SIZE + _HUB_SOURCES + _HUB_CLIENTS + _HUB_REQRATE)
|
||||
|
||||
|
||||
def validate_case(c):
|
||||
"""Stress-case validity = scenario validity + stress-specific bounds."""
|
||||
errs = S.validate_scenario(c)
|
||||
st = c.get("stress", {})
|
||||
if c["shape"] == "ds_fanout":
|
||||
if st["hubs"] < 1 or st["hubs"] > 16:
|
||||
errs.append(f"{c['id']}: ds_fanout hubs must be 1..16 "
|
||||
f"(UDPStreamer max unicast clients)")
|
||||
if c["shape"] == "hub" and (st["clients"] < 1):
|
||||
errs.append(f"{c['id']}: hub clients must be >= 1")
|
||||
# Single-datagram ceiling: keep each source's packet < 64 KB so it never
|
||||
# needs reassembly (the deliverable-packet cap).
|
||||
for src in c["sources"]:
|
||||
n_data = sum(1 for s in src["signals"] if not s["is_time"])
|
||||
elem = max((s["elements"] for s in src["signals"]
|
||||
if not s["is_time"]), default=0)
|
||||
pb = _packet_bytes(n_data, elem)
|
||||
if pb >= 65536:
|
||||
errs.append(f"{c['id']}: source {src['id']} packet {pb} B exceeds "
|
||||
f"the 64 KB single-datagram cap")
|
||||
return errs
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
ok = True
|
||||
seen_ids, seen_ws, seen_udp = set(), set(), set()
|
||||
by_axis = {}
|
||||
for c in STRESS_CASES:
|
||||
errs = validate_case(c)
|
||||
if c["id"] in seen_ids:
|
||||
errs.append("duplicate id")
|
||||
seen_ids.add(c["id"])
|
||||
if c["ws_port"] in seen_ws:
|
||||
errs.append(f"duplicate ws_port {c['ws_port']}")
|
||||
seen_ws.add(c["ws_port"])
|
||||
for src in c["sources"]:
|
||||
if src["udp_port"] in seen_udp:
|
||||
errs.append(f"duplicate udp_port {src['udp_port']}")
|
||||
seen_udp.add(src["udp_port"])
|
||||
by_axis.setdefault(c["stress"]["axis"], []).append(c["id"])
|
||||
print(f"{c['id']:20s} {c['shape']:9s} "
|
||||
f"{'OK' if not errs else errs}")
|
||||
ok = ok and not errs
|
||||
print(f"\n{len(STRESS_CASES)} stress cases across {len(by_axis)} axes, "
|
||||
f"{'ALL VALID' if ok else 'INVALID PRESENT'}")
|
||||
sys.exit(0 if ok else 1)
|
||||
Executable
+298
@@ -0,0 +1,298 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
stress_run.py — Orchestrator for the streaming-chain stress matrix (stress.py).
|
||||
|
||||
Where run_e2e.sh drives scenarios.py for *correctness*, this drives
|
||||
STRESS_CASES for *capacity*. Per case it:
|
||||
|
||||
1. generates the FileReader input + MARTe app cfg + 1..M StreamHub cfgs
|
||||
(reusing gen_data / gen_cfg unchanged — a stress case is a scenario superset),
|
||||
2. launches the server stack:
|
||||
* "hub" shape: 1 MARTe UDPStreamer feed + 1 StreamHub, then N
|
||||
parallel chain-clients (stress mode) on that one hub,
|
||||
* "ds_fanout" shape: 1 MARTe feed + M independent StreamHubs all
|
||||
subscribing to the same UDPStreamer (unicast fan-out), 1 client each,
|
||||
3. drives the clients in stress mode (liveness + sustained zoom-latency),
|
||||
4. snapshots CPU / peak-RSS of marte and every hub *while alive* (proc_perf),
|
||||
5. evaluates the per-case gate (survival + liveness hard gates; RSS + zoom-p95
|
||||
soft gates against the case's ceilings),
|
||||
|
||||
and writes stress_results.json (one record per case, carrying the axis/level for
|
||||
later scaling-curve plots). Server binaries + LD_LIBRARY_PATH come from the
|
||||
caller (run_stress.sh sources env.sh); this module only orchestrates.
|
||||
"""
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gen_cfg # noqa: E402
|
||||
import gen_data # noqa: E402
|
||||
import proc_perf # noqa: E402
|
||||
import stress as ST # noqa: E402
|
||||
|
||||
# WS port probing bases. StreamHub does not set SO_REUSEADDR, so a leftover
|
||||
# listener (e.g. an orphaned hub from an aborted run) keeps a fixed port wedged;
|
||||
# we therefore probe for genuinely-free ports at launch rather than trusting a
|
||||
# static assignment. ds_fanout hubs probe from one base, single hubs from another.
|
||||
FANOUT_WS_BASE = 9000
|
||||
SINGLE_WS_BASE = 9100
|
||||
|
||||
|
||||
def _popen(cmd, log_path):
|
||||
"""Launch cmd in its own session, stdout+stderr → log_path."""
|
||||
f = open(log_path, "w")
|
||||
return subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT,
|
||||
start_new_session=True), f
|
||||
|
||||
|
||||
def _free_ports(n, base):
|
||||
"""Find n free TCP ports on 127.0.0.1 at/above base.
|
||||
|
||||
Probed ports are closed and handed straight to the hub launch; the race
|
||||
window is negligible for this sequential localhost harness, and probing is
|
||||
what makes the suite robust to leftover wedged listeners."""
|
||||
found = []
|
||||
cand = base
|
||||
while len(found) < n and cand < base + 1000:
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
try:
|
||||
s.bind(("127.0.0.1", cand))
|
||||
found.append(cand)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
s.close()
|
||||
cand += 1
|
||||
if len(found) < n:
|
||||
raise RuntimeError(f"could not find {n} free ports from {base}")
|
||||
return found
|
||||
|
||||
|
||||
def _hub_ports(case):
|
||||
st = case["stress"]
|
||||
if case["shape"] == "ds_fanout":
|
||||
return _free_ports(st["hubs"], FANOUT_WS_BASE)
|
||||
return _free_ports(1, SINGLE_WS_BASE)
|
||||
|
||||
|
||||
def _teardown(procs_files):
|
||||
for p, _f in procs_files:
|
||||
if p.poll() is None:
|
||||
try:
|
||||
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
pass
|
||||
deadline = time.time() + 5
|
||||
for p, _f in procs_files:
|
||||
try:
|
||||
p.wait(timeout=max(0.1, deadline - time.time()))
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
os.killpg(os.getpgid(p.pid), signal.SIGKILL)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
pass
|
||||
for _p, f in procs_files:
|
||||
try:
|
||||
f.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def run_case(case, bins, work, out):
|
||||
sid = case["id"]
|
||||
st = case["stress"]
|
||||
input_bin = os.path.join(work, f"sinput_{sid}.bin")
|
||||
marte_cfg = os.path.join(work, f"sm_{sid}.cfg")
|
||||
|
||||
gen_data.write_input(case, input_bin)
|
||||
gen_cfg.write_marte_cfg(case, marte_cfg, input_bin)
|
||||
|
||||
ports = _hub_ports(case)
|
||||
hub_pf = [] # (proc, file)
|
||||
for i, p in enumerate(ports):
|
||||
hc = copy.deepcopy(case)
|
||||
hc["ws_port"] = p
|
||||
hcfg = os.path.join(work, f"sh_{sid}_{i}.cfg")
|
||||
gen_cfg.write_hub_cfg(hc, hcfg)
|
||||
proc, f = _popen([bins["hub"], "-cfg", hcfg],
|
||||
os.path.join(out, f"shub_{sid}_{i}.log"))
|
||||
hub_pf.append((proc, f))
|
||||
time.sleep(1.0)
|
||||
|
||||
marte_proc, marte_f = _popen(
|
||||
[bins["marte"], "-l", "RealTimeLoader", "-f", marte_cfg, "-s", "Running"],
|
||||
os.path.join(out, f"smarte_{sid}.log"))
|
||||
time.sleep(1.5)
|
||||
|
||||
dur, reqrate = st["dur"], st["reqrate"]
|
||||
# clean stale client outputs so a missing file means "client failed".
|
||||
for i in range(max(st["clients"], st["hubs"])):
|
||||
sp = os.path.join(work, f"stress_{sid}_c{i}.json")
|
||||
if os.path.exists(sp):
|
||||
os.remove(sp)
|
||||
|
||||
client_pf = []
|
||||
if case["shape"] == "ds_fanout":
|
||||
targets = [(i, ports[i]) for i in range(len(ports))]
|
||||
else:
|
||||
targets = [(i, ports[0]) for i in range(st["clients"])]
|
||||
for cid, wsport in targets:
|
||||
cmd = [bins["client"], "-mode", "stress",
|
||||
"-hub", f"127.0.0.1:{wsport}", "-scenario", sid,
|
||||
"-clientid", str(cid), "-dur", str(dur),
|
||||
"-reqrate", str(reqrate), "-out", work,
|
||||
"-timeout", f"{int(dur) + 90}s"]
|
||||
proc, f = _popen(cmd, os.path.join(out, f"sclient_{sid}_c{cid}.log"))
|
||||
client_pf.append((proc, f))
|
||||
|
||||
deadline = time.time() + dur + 60
|
||||
for proc, _f in client_pf:
|
||||
try:
|
||||
proc.wait(timeout=max(1.0, deadline - time.time()))
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
for _p, f in client_pf:
|
||||
f.close()
|
||||
|
||||
# survival + perf must be read while the stack is still alive.
|
||||
survival = (marte_proc.poll() is None
|
||||
and all(h.poll() is None for h, _ in hub_pf))
|
||||
perf_marte = proc_perf.snapshot(marte_proc.pid)
|
||||
perf_hubs = [proc_perf.snapshot(h.pid) for h, _ in hub_pf]
|
||||
|
||||
_teardown([(marte_proc, marte_f)] + hub_pf)
|
||||
|
||||
clients = []
|
||||
for cid, _ in targets:
|
||||
jp = os.path.join(work, f"stress_{sid}_c{cid}.json")
|
||||
if os.path.exists(jp):
|
||||
with open(jp) as f:
|
||||
clients.append(json.load(f))
|
||||
|
||||
return _evaluate(case, survival, perf_marte, perf_hubs, clients)
|
||||
|
||||
|
||||
def _rss_mb(rec):
|
||||
return rec.get("peak_rss_kb", 0) / 1024.0 if rec.get("avail") else 0.0
|
||||
|
||||
|
||||
def _evaluate(case, survival, perf_marte, perf_hubs, clients):
|
||||
g = case["stress"].get("gate", {})
|
||||
fails = []
|
||||
|
||||
if not survival:
|
||||
fails.append("server process died during run")
|
||||
if not clients:
|
||||
fails.append("no client results recorded")
|
||||
|
||||
min_frames = g.get("min_frames", 1)
|
||||
for c in clients:
|
||||
cid = c.get("clientId")
|
||||
if c.get("frames", 0) < min_frames:
|
||||
fails.append(f"c{cid} frames {c.get('frames')} < {min_frames}")
|
||||
if not c.get("monotonic", False):
|
||||
fails.append(f"c{cid} non-monotonic time axis")
|
||||
if not c.get("wallclock", False):
|
||||
fails.append(f"c{cid} non-wallclock timestamps")
|
||||
|
||||
marte_rss = _rss_mb(perf_marte)
|
||||
if "max_marte_rss_mb" in g and marte_rss > g["max_marte_rss_mb"]:
|
||||
fails.append(f"marte peakRSS {marte_rss:.0f}MB > {g['max_marte_rss_mb']}MB")
|
||||
hub_rss_max = max((_rss_mb(h) for h in perf_hubs), default=0.0)
|
||||
if "max_hub_rss_mb" in g and hub_rss_max > g["max_hub_rss_mb"]:
|
||||
fails.append(f"hub peakRSS {hub_rss_max:.0f}MB > {g['max_hub_rss_mb']}MB")
|
||||
|
||||
p95s = [c["zoomP95ms"] for c in clients if c.get("zoomCount", 0) > 0]
|
||||
zoom_p95 = max(p95s) if p95s else 0.0
|
||||
p50s = [c["zoomP50ms"] for c in clients if c.get("zoomCount", 0) > 0]
|
||||
zoom_p50 = max(p50s) if p50s else 0.0
|
||||
if "max_zoom_p95_ms" in g and zoom_p95 > g["max_zoom_p95_ms"]:
|
||||
fails.append(f"zoom p95 {zoom_p95:.0f}ms > {g['max_zoom_p95_ms']}ms")
|
||||
|
||||
st = case["stress"]
|
||||
return {
|
||||
"id": case["id"], "shape": case["shape"],
|
||||
"axis": st["axis"], "level": st["level"],
|
||||
"status": "PASS" if not fails else "FAIL",
|
||||
"survival": survival,
|
||||
"clients": len(clients),
|
||||
"min_frames": min(((c.get("frames", 0)) for c in clients), default=0),
|
||||
"marte_cpu_s": perf_marte.get("cpu_s", 0.0),
|
||||
"marte_rss_mb": round(marte_rss, 1),
|
||||
"hub_cpu_s": round(sum(h.get("cpu_s", 0.0) for h in perf_hubs), 2),
|
||||
"hub_rss_mb": round(hub_rss_max, 1),
|
||||
"zoom_count": sum(c.get("zoomCount", 0) for c in clients),
|
||||
"zoom_fail": sum(c.get("zoomFail", 0) for c in clients),
|
||||
"zoom_p50_ms": round(zoom_p50, 1),
|
||||
"zoom_p95_ms": round(zoom_p95, 1),
|
||||
"fails": fails,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description="Run the streaming-chain stress matrix")
|
||||
p.add_argument("--marte", required=True, help="MARTeApp.ex path")
|
||||
p.add_argument("--hub", required=True, help="StreamHub.ex path")
|
||||
p.add_argument("--client", required=True, help="chain-client path")
|
||||
p.add_argument("--work", required=True, help="scratch dir")
|
||||
p.add_argument("--out", required=True, help="report/log dir")
|
||||
p.add_argument("--only", default="", help="run a single case id")
|
||||
p.add_argument("--axis", default="", help="run only cases on this axis")
|
||||
args = p.parse_args()
|
||||
|
||||
os.makedirs(args.work, exist_ok=True)
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
bins = {"marte": args.marte, "hub": args.hub, "client": args.client}
|
||||
|
||||
cases = ST.STRESS_CASES
|
||||
if args.only:
|
||||
cases = [c for c in cases if c["id"] == args.only]
|
||||
if args.axis:
|
||||
cases = [c for c in cases if c["stress"]["axis"] == args.axis]
|
||||
if not cases:
|
||||
print("no stress cases selected", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
results = []
|
||||
for c in cases:
|
||||
errs = ST.validate_case(c)
|
||||
if errs:
|
||||
print(f"══ {c['id']}: INVALID {errs} ══")
|
||||
results.append({"id": c["id"], "axis": c["stress"]["axis"],
|
||||
"level": c["stress"]["level"], "status": "FAIL",
|
||||
"fails": errs})
|
||||
continue
|
||||
print(f"\n══ stress {c['id']} ({c['shape']} {c['stress']['axis']}="
|
||||
f"{c['stress']['level']}) ══")
|
||||
rec = run_case(c, bins, args.work, args.out)
|
||||
results.append(rec)
|
||||
tag = rec["status"]
|
||||
print(f" {tag} frames>={rec.get('min_frames')} "
|
||||
f"marteCPU={rec.get('marte_cpu_s', 0):.1f}s "
|
||||
f"marteRSS={rec.get('marte_rss_mb', 0):.0f}MB "
|
||||
f"hubRSS={rec.get('hub_rss_mb', 0):.0f}MB "
|
||||
f"zoom p50/p95={rec.get('zoom_p50_ms', 0):.0f}/"
|
||||
f"{rec.get('zoom_p95_ms', 0):.0f}ms")
|
||||
if rec.get("fails"):
|
||||
for fmsg in rec["fails"]:
|
||||
print(f" - {fmsg}")
|
||||
|
||||
overall = bool(results) and all(r["status"] == "PASS" for r in results)
|
||||
doc = {"overall": "PASS" if overall else "FAIL", "cases": results}
|
||||
rp = os.path.join(args.out, "stress_results.json")
|
||||
with open(rp, "w") as f:
|
||||
json.dump(doc, f, indent=2)
|
||||
npass = sum(r["status"] == "PASS" for r in results)
|
||||
print(f"\nstress_results.json: {npass}/{len(results)} pass → {doc['overall']}")
|
||||
sys.exit(0 if overall else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,188 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
tests_py.py — Unit tests for the streaming-chain E2E Python framework.
|
||||
|
||||
Run directly (``python3 -m unittest tests_py``) or, for coverage, via collect.py
|
||||
which wraps it in ``coverage run``. These exercise the pure logic of the
|
||||
generators, the waveform oracle, the config emitter, the RCV1 reader and the
|
||||
/proc perf sampler — i.e. everything the orchestrator depends on, without needing
|
||||
a live MARTe/StreamHub stack.
|
||||
"""
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import scenarios as S
|
||||
import gen_data as G
|
||||
import gen_cfg as C
|
||||
import validate_waveform as V
|
||||
import proc_perf as P
|
||||
|
||||
|
||||
class TestScenarios(unittest.TestCase):
|
||||
def test_all_starter_scenarios_valid(self):
|
||||
for s in S.SCENARIOS:
|
||||
self.assertEqual(S.validate_scenario(s), [], f"{s['id']} invalid")
|
||||
|
||||
def test_seamless_loop_constraint(self):
|
||||
s = {
|
||||
"id": "x", "network": "unicast", "publishing": "Strict",
|
||||
"ratio": None, "min_refresh_hz": None, "max_payload": 1400,
|
||||
"ws_port": 9000,
|
||||
"sources": [{"id": "s", "udp_port": 1, "data_port": None,
|
||||
"multicast_group": None,
|
||||
"signals": [S._sig("Bad", "float32", freq=3.0)]}],
|
||||
"oracle": "analytic", "client_checks": ["live"], "trig_signal": None,
|
||||
}
|
||||
errs = S.validate_scenario(s)
|
||||
self.assertTrue(any("multiple of LOOP_HZ" in e for e in errs), errs)
|
||||
s["sources"][0]["signals"][0]["freq"] = S.LOOP_HZ * 2
|
||||
self.assertEqual(S.validate_scenario(s), [])
|
||||
|
||||
def test_bad_quant_rejected(self):
|
||||
s = {
|
||||
"id": "x", "network": "unicast", "publishing": "Strict",
|
||||
"ratio": None, "min_refresh_hz": None, "max_payload": 1400,
|
||||
"ws_port": 9000,
|
||||
"sources": [{"id": "s", "udp_port": 1, "data_port": None,
|
||||
"multicast_group": None,
|
||||
"signals": [S._sig("Q", "int32", quant="uint8",
|
||||
range_min=0, range_max=1, formula="ramp")]}],
|
||||
"oracle": "analytic", "client_checks": ["live"], "trig_signal": None,
|
||||
}
|
||||
errs = S.validate_scenario(s)
|
||||
self.assertTrue(any("quant only on float" in e for e in errs), errs)
|
||||
|
||||
|
||||
class TestGenData(unittest.TestCase):
|
||||
def test_ground_truth_shapes(self):
|
||||
gt = G.build_ground_truth(S.SCENARIOS[1]) # s02: TimeArr+Wave, 100 elem
|
||||
wave = gt["src:Wave"]
|
||||
self.assertEqual(wave["elements"], 100)
|
||||
self.assertEqual(wave["v"].size, S.NUM_ROWS * 100)
|
||||
self.assertEqual(wave["t"].size, S.NUM_ROWS * 100)
|
||||
|
||||
def test_binary_roundtrip(self):
|
||||
sc = S.SCENARIOS[0] # s01: Counter(uint32)+Sine(float32)
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p = os.path.join(d, "in.bin")
|
||||
gt = G.write_input(sc, p)
|
||||
cols = V.read_marte_binary(p)
|
||||
self.assertIn("Counter", cols)
|
||||
self.assertIn("Sine", cols)
|
||||
# counter is row index → first/last match ground truth
|
||||
self.assertEqual(cols["Counter"].reshape(-1)[0], gt["src:Counter"]["v"][0])
|
||||
self.assertEqual(int(cols["Counter"].reshape(-1)[5]), 5)
|
||||
|
||||
|
||||
class TestOracle(unittest.TestCase):
|
||||
def _gt(self, **kw):
|
||||
t = np.linspace(0, 1, 200)
|
||||
base = {"v": np.sin(2 * np.pi * 5 * t), "type": "float32", "quant": "none",
|
||||
"formula": "sine", "freq": 5.0, "range_min": -1.0, "range_max": 1.0,
|
||||
"elements": 1, "rows": 200, "is_time": False, "t": t,
|
||||
"dt": t[1] - t[0]}
|
||||
base.update(kw)
|
||||
return base, t
|
||||
|
||||
def test_quant_tol_is_one_level(self):
|
||||
gt, _ = self._gt(quant="uint16", range_min=-5.0, range_max=5.0)
|
||||
tol, step = V._tol(gt)
|
||||
self.assertAlmostEqual(step, 10.0 / 65535, places=9)
|
||||
self.assertGreaterEqual(tol, step) # one full level, not half
|
||||
|
||||
def test_good_passes_bad_fails(self):
|
||||
gt, t = self._gt(quant="uint16")
|
||||
_, step = V._tol(gt)
|
||||
good = gt["v"] + (np.random.rand(200) - 0.5) * step
|
||||
bad = gt["v"] + (np.random.rand(200) - 0.5) * step * 50
|
||||
self.assertTrue(V.compare_signal(gt, t, good)["pass"])
|
||||
self.assertFalse(V.compare_signal(gt, t, bad)["pass"])
|
||||
|
||||
def test_wrong_frequency_fails_shape(self):
|
||||
gt, t = self._gt()
|
||||
wrong = np.sin(2 * np.pi * 50 * t) # 10x frequency
|
||||
m = V.compare_signal(gt, t, wrong)
|
||||
self.assertFalse(m["shape_ok"], m)
|
||||
|
||||
def test_integer_offgrid_fails(self):
|
||||
gt, t = self._gt(type="uint32", quant="none",
|
||||
v=np.arange(200, dtype=np.float64), formula="counter")
|
||||
self.assertTrue(V.compare_signal(gt, t, np.arange(50, 150, dtype=float))["pass"])
|
||||
self.assertFalse(V.compare_signal(gt, t, np.array([1.5, 999.0]))["pass"])
|
||||
|
||||
|
||||
class TestRCV1(unittest.TestCase):
|
||||
def test_read_received(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p = os.path.join(d, "r.bin")
|
||||
t = [0.0, 0.1, 0.2]
|
||||
v = [1.0, 2.0, 3.0]
|
||||
buf = bytearray(b"RCV1")
|
||||
buf += struct.pack("<I", 1)
|
||||
key = b"src:Sig"
|
||||
buf += struct.pack("<H", len(key)) + key + struct.pack("<I", len(t))
|
||||
for x in t:
|
||||
buf += struct.pack("<d", x)
|
||||
for x in v:
|
||||
buf += struct.pack("<d", x)
|
||||
open(p, "wb").write(buf)
|
||||
r = V.read_received(p)
|
||||
self.assertIn("src:Sig", r)
|
||||
tt, vv = r["src:Sig"]
|
||||
self.assertEqual(list(vv), v)
|
||||
|
||||
|
||||
class TestGenCfg(unittest.TestCase):
|
||||
def test_tap_routes_through_ddb(self):
|
||||
sc = S.SCENARIOS[1] # s02, oracle=both
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
mc = os.path.join(d, "m.cfg")
|
||||
C.write_marte_cfg(sc, mc, os.path.join(d, "in.bin"),
|
||||
tap_bin=os.path.join(d, "tap.bin"))
|
||||
cfg = open(mc).read()
|
||||
self.assertIn("StreamGAM_src", cfg)
|
||||
self.assertIn("TapGAM", cfg)
|
||||
# TapGAM must read from the DDB (prefixed name), not the FileReader
|
||||
self.assertIn("src_Wave", cfg)
|
||||
tap_block = cfg[cfg.index("+TapGAM"):]
|
||||
tap_in = tap_block[tap_block.index("InputSignals"):tap_block.index("OutputSignals")]
|
||||
self.assertNotIn("FileReaderDS", tap_in)
|
||||
|
||||
def test_no_tap_direct(self):
|
||||
sc = S.SCENARIOS[0] # s01, oracle=analytic
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
mc = os.path.join(d, "m.cfg")
|
||||
C.write_marte_cfg(sc, mc, os.path.join(d, "in.bin"))
|
||||
cfg = open(mc).read()
|
||||
self.assertNotIn("TapGAM", cfg)
|
||||
self.assertIn("ReaderGAM_src", cfg)
|
||||
|
||||
def test_hub_cfg_has_ports(self):
|
||||
sc = S.SCENARIOS[0]
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
hc = os.path.join(d, "h.cfg")
|
||||
C.write_hub_cfg(sc, hc)
|
||||
cfg = open(hc).read()
|
||||
self.assertIn(f"WSPort = {sc['ws_port']}", cfg)
|
||||
self.assertIn(f"Port = {sc['sources'][0]['udp_port']}", cfg)
|
||||
|
||||
|
||||
class TestProcPerf(unittest.TestCase):
|
||||
def test_snapshot_self(self):
|
||||
rec = P.snapshot(os.getpid())
|
||||
self.assertTrue(rec["avail"])
|
||||
self.assertIn("cpu_s", rec)
|
||||
self.assertGreaterEqual(rec["cpu_s"], 0.0)
|
||||
|
||||
def test_snapshot_missing(self):
|
||||
self.assertFalse(P.snapshot(2 ** 30).get("avail", False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,325 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
validate_waveform.py — Per-effect waveform validator for the streaming-chain E2E.
|
||||
|
||||
Compares the client-recorded stream (``received_<id>.bin``, RCV1 format written
|
||||
by chain-client) against the analytic ground truth (rebuilt from gen_data) and,
|
||||
when present, the fed-reference tap (``tap_<id>.bin``, MARTe binary). It also
|
||||
rolls in the client behavioural checks (``checks_<id>.json``) and emits
|
||||
``metrics_<id>.json`` with an overall pass/fail (exit 0/1).
|
||||
|
||||
Oracle (per signal)
|
||||
-------------------
|
||||
* **Fidelity** (always): every received value must lie within ``tol`` of *some*
|
||||
ground-truth value. ``tol`` is 0 (bit-exact) for un-quantised integers, a tiny
|
||||
float epsilon for un-quantised floats, and ``quant_step/2 + 1e-6·range`` for
|
||||
quantised floats. Catches type corruption and out-of-range quantisation.
|
||||
* **Shape** (sine signals, ≥8 points): least-squares fit of
|
||||
``a·sin(ωt)+b·cos(ωt)+c`` at the scenario frequency ω=2πf. A high correlation
|
||||
(≥0.99) and low normalised RMSE confirm the received waveform is the expected
|
||||
sinusoid at the expected frequency (the fit recovers the unknown wall-clock
|
||||
phase offset automatically). nRMSE tolerance is relaxed by the quant step.
|
||||
* **Fed reference** (when ``--tap`` given): each received value must be within
|
||||
``tol`` of some tap value too.
|
||||
* **Continuity** (always, ≥10 points): a stream that stalls (client falling
|
||||
behind, hub failing to flush a window, ...) can still pass fidelity —
|
||||
whatever few samples *did* arrive still match the ground truth — while the
|
||||
plot shows gaping holes. Flags any inter-sample gaps that are >10x the
|
||||
median spacing and fails when their *summed* duration exceeds 5% of the
|
||||
capture span. Calibrated against the full scenario matrix: healthy streams
|
||||
(bursty per-tick live pushes, decimation, fragmentation, multicast, ...) top
|
||||
out at 0.7% outlier-gap time; a stalled stream showed 55-91%.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import scenarios as S # noqa: E402
|
||||
import gen_data as G # noqa: E402
|
||||
|
||||
CODE_TO_TYPE = {v: k for k, v in S.TYPE_CODES.items()}
|
||||
|
||||
|
||||
# ── readers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def read_received(path):
|
||||
"""Read RCV1 → {key: (t ndarray, v ndarray)}."""
|
||||
with open(path, "rb") as f:
|
||||
data = f.read()
|
||||
if data[:4] != b"RCV1":
|
||||
raise ValueError(f"{path}: bad magic")
|
||||
off = 4
|
||||
nsig = struct.unpack_from("<I", data, off)[0]
|
||||
off += 4
|
||||
out = {}
|
||||
for _ in range(nsig):
|
||||
klen = struct.unpack_from("<H", data, off)[0]
|
||||
off += 2
|
||||
key = data[off:off + klen].decode()
|
||||
off += klen
|
||||
n = struct.unpack_from("<I", data, off)[0]
|
||||
off += 4
|
||||
t = np.frombuffer(data, "<f8", n, off).copy()
|
||||
off += 8 * n
|
||||
v = np.frombuffer(data, "<f8", n, off).copy()
|
||||
off += 8 * n
|
||||
out[key] = (t, v)
|
||||
return out
|
||||
|
||||
|
||||
def read_marte_binary(path):
|
||||
"""Read a MARTe binary file → {name: ndarray[n_rows, elements]} (typed)."""
|
||||
with open(path, "rb") as f:
|
||||
data = f.read()
|
||||
ns = struct.unpack_from("<I", data, 0)[0]
|
||||
off = 4
|
||||
descs = []
|
||||
for _ in range(ns):
|
||||
tc = struct.unpack_from("<H", data, off)[0]
|
||||
name = data[off + 2:off + 34].rstrip(b"\0").decode(errors="replace")
|
||||
ne = struct.unpack_from("<I", data, off + 34)[0]
|
||||
descs.append((name, tc, ne))
|
||||
off += 38
|
||||
row_bytes = sum(struct.calcsize(_npfmt(tc)) * ne for _, tc, ne in descs)
|
||||
nrows = (len(data) - off) // row_bytes if row_bytes else 0
|
||||
cols = {name: [] for name, _, _ in descs}
|
||||
for r in range(nrows):
|
||||
for name, tc, ne in descs:
|
||||
dt = np.dtype(S.NP_DTYPE[CODE_TO_TYPE[tc]])
|
||||
vals = np.frombuffer(data, dt, ne, off)
|
||||
cols[name].append(vals.astype(np.float64))
|
||||
off += dt.itemsize * ne
|
||||
return {name: np.array(cols[name]) for name, _, _ in descs}
|
||||
|
||||
|
||||
def _npfmt(tc):
|
||||
return {"<i1": "b", "<u1": "B", "<i2": "h", "<u2": "H", "<i4": "i",
|
||||
"<u4": "I", "<i8": "q", "<u8": "Q", "<f4": "f",
|
||||
"<f8": "d"}[S.NP_DTYPE[CODE_TO_TYPE[tc]]]
|
||||
|
||||
|
||||
# ── metrics ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def _tol(gt):
|
||||
if gt["quant"] and gt["quant"] != "none":
|
||||
levels = S.QUANT_LEVELS[gt["quant"]]
|
||||
rng = gt["range_max"] - gt["range_min"]
|
||||
step = rng / levels
|
||||
# One full quantisation level: the correctness bound for lossy quant when
|
||||
# the encode rounding convention (round vs truncate) is unknown. Gross
|
||||
# corruption is many levels off; a faithful round-trip is ≤1 level.
|
||||
return step + 1e-6 * abs(rng), step
|
||||
if gt["type"] in S.FLOAT_TYPES:
|
||||
return 1e-3, 0.0 # float round-trip epsilon
|
||||
return 0.5, 0.0 # integer: rounding-exact (within 0.5)
|
||||
|
||||
|
||||
def nearest_err(recv_v, truth_v):
|
||||
"""Max distance from each received value to the nearest truth value."""
|
||||
sv = np.unique(np.sort(truth_v.astype(np.float64)))
|
||||
idx = np.searchsorted(sv, recv_v)
|
||||
idx = np.clip(idx, 1, len(sv) - 1) if len(sv) > 1 else np.zeros_like(idx)
|
||||
lo = sv[np.clip(idx - 1, 0, len(sv) - 1)]
|
||||
hi = sv[np.clip(idx, 0, len(sv) - 1)]
|
||||
d = np.minimum(np.abs(recv_v - lo), np.abs(recv_v - hi))
|
||||
return float(np.max(d)) if d.size else 0.0
|
||||
|
||||
|
||||
def gap_check(t_recv, outlier_mult=10.0, max_outlier_frac=0.05):
|
||||
"""Detect large discontiguous holes in a received time series.
|
||||
|
||||
A handful of gaps a few times the median spacing are normal (bursty
|
||||
per-tick live pushes, decimation, LTTB). Returns (ok, gap_frac, n_gaps,
|
||||
max_gap): ``gap_frac`` is the fraction of the total capture span consumed
|
||||
by gaps larger than ``outlier_mult`` times the median inter-sample gap;
|
||||
when that adds up to more than ``max_outlier_frac`` of the whole capture,
|
||||
the stream stalled/dropped a chunk rather than merely being decimated.
|
||||
"""
|
||||
if t_recv.size < 10:
|
||||
return True, 0.0, 0, 0.0
|
||||
t = np.sort(t_recv.astype(np.float64))
|
||||
dt = np.diff(t)
|
||||
span = float(t[-1] - t[0])
|
||||
med = float(np.median(dt))
|
||||
if span <= 0.0 or med <= 0.0:
|
||||
return True, 0.0, 0, 0.0
|
||||
outliers = dt[dt > outlier_mult * med]
|
||||
gap_frac = float(outliers.sum() / span)
|
||||
return gap_frac <= max_outlier_frac, gap_frac, int(outliers.size), float(dt.max())
|
||||
|
||||
|
||||
def sine_shape(t, v, freq):
|
||||
"""Return (corr, nrmse, amp_fit) for a sinusoid fit at ``freq``."""
|
||||
w = 2.0 * np.pi * freq
|
||||
A = np.column_stack([np.sin(w * t), np.cos(w * t), np.ones_like(t)])
|
||||
coef, *_ = np.linalg.lstsq(A, v, rcond=None)
|
||||
fit = A @ coef
|
||||
span = float(np.max(v) - np.min(v)) or 1.0
|
||||
nrmse = float(np.sqrt(np.mean((v - fit) ** 2)) / span)
|
||||
corr = float(np.corrcoef(v, fit)[0, 1]) if np.std(v) > 0 else 1.0
|
||||
amp = float(np.hypot(coef[0], coef[1]))
|
||||
return corr, nrmse, amp
|
||||
|
||||
|
||||
def compare_signal(gt, t_recv, v_recv, tap_v=None):
|
||||
tol, step = _tol(gt)
|
||||
truth_v = gt["v"].astype(np.float64)
|
||||
m = {
|
||||
"type": gt["type"], "quant": gt["quant"], "formula": gt["formula"],
|
||||
"n_truth": int(truth_v.size), "n_recv": int(v_recv.size),
|
||||
"quant_step": step, "tol": tol,
|
||||
}
|
||||
if v_recv.size == 0:
|
||||
m["pass"] = False
|
||||
m["reason"] = "no received samples"
|
||||
return m
|
||||
|
||||
max_err = nearest_err(v_recv, truth_v)
|
||||
m["max_abs_err"] = max_err
|
||||
fidelity_ok = max_err <= tol
|
||||
m["fidelity_ok"] = bool(fidelity_ok)
|
||||
|
||||
gap_ok, gap_frac, n_gaps, max_gap = gap_check(t_recv)
|
||||
m.update(gap_ok=bool(gap_ok), gap_frac=round(gap_frac, 4),
|
||||
n_gaps=n_gaps, max_gap=max_gap)
|
||||
if not gap_ok:
|
||||
m["reason"] = (f"data hole: {gap_frac:.1%} of capture span in "
|
||||
f"{n_gaps} gaps >10x median spacing (max={max_gap:.4g}s)")
|
||||
|
||||
shape_ok = True
|
||||
if gt["formula"] == "sine" and v_recv.size >= 8 and gt["freq"]:
|
||||
corr, nrmse, amp = sine_shape(t_recv, v_recv, gt["freq"])
|
||||
# Shape is a *gross frequency-sanity gate* plus a *tracked quality
|
||||
# metric*, not a tight correctness gate. Signal values are bit-faithful
|
||||
# (the fidelity oracle proves that); the gap from a perfect fit is
|
||||
# almost entirely x-axis timestamp jitter: the hub assigns wall-clock
|
||||
# times without per-sample calibration (Phase-A) and the FULL_ARRAY
|
||||
# packed-timestamp decode is incomplete (Phase-A4) — both pending. For a
|
||||
# correct sinusoid that yields corr ~0.82-0.98 (more for arrays); a
|
||||
# wrong-frequency or corrupted signal collapses to corr ~0.00. So the
|
||||
# gate (corr>=0.5, nRMSE<=0.30) reliably rejects gross corruption with a
|
||||
# wide margin, while corr/nRMSE are recorded so the report can trend
|
||||
# them toward 1.0/0.0 as the timestamping work lands (progression).
|
||||
nrmse_tol = 0.30 + (step / (gt["range_max"] - gt["range_min"])
|
||||
if gt["quant"] != "none" else 0.0)
|
||||
shape_ok = corr >= 0.5 and nrmse <= nrmse_tol
|
||||
m.update(corr=corr, nrmse=nrmse, amp_fit=amp,
|
||||
nrmse_tol=nrmse_tol, shape_ok=bool(shape_ok),
|
||||
shape_gate="gross")
|
||||
|
||||
fed_ok = True
|
||||
if tap_v is not None and tap_v.size:
|
||||
fed_err = nearest_err(v_recv, tap_v.astype(np.float64))
|
||||
fed_ok = fed_err <= tol
|
||||
m.update(fed_err=fed_err, fed_ok=bool(fed_ok))
|
||||
|
||||
m["pass"] = bool(fidelity_ok and shape_ok and fed_ok and gap_ok)
|
||||
return m
|
||||
|
||||
|
||||
# ── driver ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def validate(scenario, received, tap, checks):
|
||||
gt = G.build_ground_truth(scenario)
|
||||
recv = read_received(received)
|
||||
tap_cols = read_marte_binary(tap) if tap and os.path.exists(tap) else None
|
||||
|
||||
sigs = {}
|
||||
overall = True
|
||||
for key, (t, v) in sorted(recv.items()):
|
||||
# key is "src:sig" or "src:sig[i]" (per-element array push)
|
||||
base = key.split("[")[0]
|
||||
if base not in gt:
|
||||
sigs[key] = {"pass": True, "note": "no ground truth (skipped)",
|
||||
"n_recv": int(v.size)}
|
||||
continue
|
||||
tap_v = None
|
||||
if tap_cols is not None:
|
||||
sig_name = base.split(":", 1)[1]
|
||||
if sig_name in tap_cols:
|
||||
tap_v = tap_cols[sig_name].reshape(-1)
|
||||
m = compare_signal(gt[base], t, v, tap_v)
|
||||
sigs[key] = m
|
||||
overall = overall and m.get("pass", False)
|
||||
|
||||
# roll in client checks
|
||||
client = {}
|
||||
if checks and os.path.exists(checks):
|
||||
with open(checks) as f:
|
||||
client = json.load(f)
|
||||
live_ok = client.get("live", {}).get("ok", False)
|
||||
zoom_ok = all(z.get("inrange", False) for z in client.get("zoom", [])) \
|
||||
if client.get("zoom") else True
|
||||
win = client.get("window", {})
|
||||
window_ok = win.get("ok", True) if win.get("returned", 0) else True
|
||||
trig = client.get("trigger", [])
|
||||
trig_ok = all(t.get("fired", False) and t.get("windowOk", False)
|
||||
for t in trig) if trig else True
|
||||
overall = overall and live_ok and zoom_ok and window_ok and trig_ok
|
||||
client["_rollup"] = {"live_ok": live_ok, "zoom_ok": zoom_ok,
|
||||
"window_ok": window_ok, "trigger_ok": trig_ok}
|
||||
|
||||
return {"scenario": scenario["id"], "signals": sigs,
|
||||
"client": client, "pass": bool(overall)}
|
||||
|
||||
|
||||
def _selftest():
|
||||
import math
|
||||
t = np.linspace(0, 1, 200)
|
||||
truth = np.sin(2 * np.pi * 3 * t)
|
||||
gt = {"v": truth, "type": "float32", "quant": "uint16", "formula": "sine",
|
||||
"freq": 3.0, "range_min": -1.0, "range_max": 1.0, "elements": 1,
|
||||
"rows": 200, "is_time": False, "t": t, "dt": t[1] - t[0]}
|
||||
_, step = _tol(gt)
|
||||
good = truth + (np.random.rand(200) - 0.5) * step # ≤ step/2
|
||||
bad = truth + (np.random.rand(200) - 0.5) * step * 8 # ≫ step
|
||||
mg = compare_signal(gt, t, good)
|
||||
mb = compare_signal(gt, t, bad)
|
||||
assert mg["pass"], f"good should pass: {mg}"
|
||||
assert not mb["pass"], f"bad should fail: {mb}"
|
||||
# bit-exact integer
|
||||
gi = dict(gt); gi.update(type="uint32", quant="none",
|
||||
v=np.arange(200, dtype=np.float64), formula="counter")
|
||||
mi = compare_signal(gi, t, np.arange(50, 150, dtype=np.float64))
|
||||
assert mi["pass"], f"int subset should pass: {mi}"
|
||||
mi2 = compare_signal(gi, t, np.array([1.5, 250.0]))
|
||||
assert not mi2["pass"], f"int off-grid should fail: {mi2}"
|
||||
print("selftest OK")
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description="Validate received waveform")
|
||||
p.add_argument("--selftest", action="store_true")
|
||||
p.add_argument("--scenario")
|
||||
p.add_argument("--received")
|
||||
p.add_argument("--tap", default=None)
|
||||
p.add_argument("--checks", default=None)
|
||||
p.add_argument("--out", default=None)
|
||||
args = p.parse_args()
|
||||
|
||||
if args.selftest:
|
||||
_selftest()
|
||||
sys.exit(0)
|
||||
|
||||
sc = next((s for s in S.SCENARIOS if s["id"] == args.scenario), None)
|
||||
if sc is None:
|
||||
print(f"unknown scenario {args.scenario}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
res = validate(sc, args.received, args.tap, args.checks)
|
||||
if args.out:
|
||||
with open(args.out, "w") as f:
|
||||
json.dump(res, f, indent=2)
|
||||
print(f"{sc['id']}: {'PASS' if res['pass'] else 'FAIL'}")
|
||||
for k, m in res["signals"].items():
|
||||
print(f" {k}: pass={m.get('pass')} "
|
||||
f"err={m.get('max_abs_err','-')} corr={m.get('corr','-')}")
|
||||
sys.exit(0 if res["pass"] else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user