diff --git a/Test/E2E/suite/E2E_Report.typ b/Test/E2E/suite/E2E_Report.typ index c98c437..08b5227 100644 --- a/Test/E2E/suite/E2E_Report.typ +++ b/Test/E2E/suite/E2E_Report.typ @@ -323,6 +323,79 @@ MARTe2 processes, plus sustained client throughput (recorded samples ÷ duration #v(6pt) ] +// ── per-kind sections (direct/recorder/debug/tcplogger) ───────────────────── +// Raw scenario records here come straight from results.json (not e2e.scenarios' +// reshaping), so they only carry id/kind/status/known_issue/metrics — no +// waveform-fidelity breakdown (already covered, where applicable, in the +// Scenarios section above for chain-kind scenarios; these kinds run through +// dedicated non-chain harnesses in run_e2e.sh). +#let kind_table(title, block) = { + [= #title] + [#block.n_pass/#block.n_total passed.] + v(4pt) + if block.n_total == 0 { + text(fill: neutral)[_No scenarios of this kind in this run._] + } else { + table( + columns: (1.4fr, 0.8fr, 2fr), + align: (left, center, left), + stroke: 0.4pt + rgb("#d0d7de"), + inset: 5pt, + table.header([*Scenario*], [*Status*], [*Known issue*]), + ..block.scenarios.map(s => ( + [#s.id], + status_badge(s.status), + if s.at("known_issue", default: none) != none { + text(size: 8pt, fill: rgb("#7d4e00"))[#s.known_issue] + } else { text(fill: neutral)[—] }, + )).flatten() + ) + } +} + +#kind_table("Direct Round-Trip (UDPStreamer ↔ UDPStreamerClient)", data.direct) +#kind_table("Recorder (BinaryRecorder disk output)", data.recorder) +#kind_table("Debug Service E2E", data.debug) +#kind_table("TCPLogger E2E", data.tcplogger) + +// ── stress tests ───────────────────────────────────────────────────────────── += Stress Tests +#if data.stress == none [ + _Not run this session._ +] else [ + #align(center, status_badge(data.stress.overall) + h(8pt) + text(size: 11pt)[ + #hl.at("stress_pass", default: 0) passed · + #hl.at("stress_fail", default: 0) failed + ]) + #v(4pt) + #for (axis, cases) in data.stress.by_axis [ + == Axis: #raw(axis) + #table( + columns: (0.8fr, 0.8fr, 1fr, 1fr, 1fr, 1fr), + align: (right, center, right, right, right, right), + stroke: 0.4pt + rgb("#d0d7de"), + inset: 4pt, + table.header([*Level*], [*Status*], [*Hub RSS (MB)*], [*MARTe RSS (MB)*], + [*Zoom p50 (ms)*], [*Zoom p95 (ms)*]), + ..cases.map(c => ( + [#c.at("level", default: "n/a")], + status_badge(c.at("status", default: "?")), + fnum(c.at("hub_rss_mb", default: none), digits: 1), + fnum(c.at("marte_rss_mb", default: none), digits: 1), + fnum(c.at("zoom_p50_ms", default: none), digits: 1), + fnum(c.at("zoom_p95_ms", default: none), digits: 1), + )).flatten() + ) + ] + #if data.stress_plots.len() > 0 [ + #v(4pt) + == Scaling curves + #grid(columns: 2, gutter: 8pt, + ..data.stress_plots.map(p => image(p, width: 100%)) + ) + ] +] + // ── trend plots ────────────────────────────────────────────────────────────── #if data.trend_plots.len() > 0 [ = Trends over runs diff --git a/Test/E2E/suite/report_build.py b/Test/E2E/suite/report_build.py index 21917ba..331aeb3 100644 --- a/Test/E2E/suite/report_build.py +++ b/Test/E2E/suite/report_build.py @@ -139,6 +139,23 @@ def build_e2e(results, work): } +def build_by_kind(results, kind): + """Filter raw scenario records (as written by run_e2e.sh's results.json + aggregation) down to a single scenario `kind` (direct/recorder/debug/ + tcplogger), with a small pass/fail rollup for the report's headline KPIs + and per-kind section.""" + scenarios = [s for s in results.get("scenarios", []) if s.get("kind") == kind] + n_pass = sum(1 for s in scenarios if s.get("status") == "PASS") + n_fail = sum(1 for s in scenarios if s.get("status") == "FAIL") + return { + "kind": kind, + "n_pass": n_pass, + "n_fail": n_fail, + "n_total": len(scenarios), + "scenarios": scenarios, + } + + def headline(e2e, ut, cov): cov_by = {c["name"]: c.get("pct") for c in cov.get("languages", [])} t = ut.get("totals", {}) @@ -236,12 +253,117 @@ def trend_plots(history, out): return made +# ── stress (Test/E2E/suite/stress.py + stress_run.py) ─────────────────────── +_STRESS_LABELS = { + "stress_pass": "Stress cases passed", + "stress_fail": "Stress cases failed", + "stress_max_hub_rss_mb": "Stress max hub RSS (MB)", + "stress_max_marte_rss_mb": "Stress max MARTe RSS (MB)", + "stress_max_zoom_p95_ms": "Stress max zoom p95 (ms)", +} +_STRESS_DIRECTION = { + "stress_pass": True, "stress_fail": False, + "stress_max_hub_rss_mb": False, "stress_max_marte_rss_mb": False, + "stress_max_zoom_p95_ms": False, +} +_DIRECTION.update({ + "direct_pass": True, "direct_fail": False, + "recorder_pass": True, "recorder_fail": False, + "debug_pass": True, "debug_fail": False, + "tcplogger_pass": True, "tcplogger_fail": False, +}) +_DIRECTION.update(_STRESS_DIRECTION) +_LABELS.update({ + "direct_pass": "Direct scenarios passed", "direct_fail": "Direct scenarios failed", + "recorder_pass": "Recorder scenarios passed", "recorder_fail": "Recorder scenarios failed", + "debug_pass": "Debug scenarios passed", "debug_fail": "Debug scenarios failed", + "tcplogger_pass": "TCPLogger scenarios passed", "tcplogger_fail": "TCPLogger scenarios failed", +}) +_LABELS.update(_STRESS_LABELS) + + +def build_stress(sr): + """Shape stress_results.json into the report's stress block (+ by_axis).""" + cases = sr.get("cases", []) or [] + by_axis = {} + for c in cases: + by_axis.setdefault(c.get("axis", "?"), []).append(c) + for axis in by_axis: + by_axis[axis].sort(key=lambda c: c.get("level", 0)) + return {"overall": sr.get("overall", "FAIL"), "cases": cases, + "by_axis": by_axis} + + +def stress_headline(stress): + cases = stress.get("cases", []) or [] + return { + "stress_pass": sum(1 for c in cases if c.get("status") == "PASS"), + "stress_fail": sum(1 for c in cases if c.get("status") == "FAIL"), + "stress_max_hub_rss_mb": max((c.get("hub_rss_mb", 0) or 0 + for c in cases), default=0.0), + "stress_max_marte_rss_mb": max((c.get("marte_rss_mb", 0) or 0 + for c in cases), default=0.0), + "stress_max_zoom_p95_ms": max((c.get("zoom_p95_ms", 0) or 0 + for c in cases), default=0.0), + } + + +_STRESS_AXIS_METRICS = { + "ds_signal_elements": [("MARTe RSS (MB)", "marte_rss_mb"), + ("hub RSS (MB)", "hub_rss_mb")], + "hub_signal_elements": [("hub RSS (MB)", "hub_rss_mb"), + ("hub CPU (s)", "hub_cpu_s")], + "ds_signal_count": [("MARTe RSS (MB)", "marte_rss_mb"), + ("MARTe CPU (s)", "marte_cpu_s")], + "hub_source_count": [("hub RSS (MB)", "hub_rss_mb"), + ("MARTe RSS (MB)", "marte_rss_mb")], + "hub_ws_clients": [("hub RSS (MB)", "hub_rss_mb"), + ("hub CPU (s)", "hub_cpu_s")], + "ds_subscriber_hubs": [("hub RSS (MB)", "hub_rss_mb"), + ("MARTe CPU (s)", "marte_cpu_s")], + "hub_zoom_reqrate_hz": [("zoom p95 (ms)", "zoom_p95_ms"), + ("zoom p50 (ms)", "zoom_p50_ms")], +} + + +def stress_plots(by_axis, out): + """One scaling-curve PNG per axis: level (x) vs the axis's metrics (y).""" + made = [] + for axis, cases in by_axis.items(): + series = _STRESS_AXIS_METRICS.get( + axis, [("hub RSS (MB)", "hub_rss_mb"), ("MARTe RSS (MB)", "marte_rss_mb")]) + xs = [c.get("level") for c in cases] + fig, ax = plt.subplots(figsize=(7, 3)) + plotted = False + for lbl, field in series: + ys = [c.get(field) for c in cases] + if all((v is None or v == 0) for v in ys): + continue + ax.plot(xs, ys, "o-", label=lbl) + plotted = True + if not plotted: + plt.close(fig) + continue + ax.set_title(f"Scaling: {axis}") + ax.set_xlabel("load level") + ax.set_ylabel("value") + ax.grid(alpha=0.3) + ax.legend(fontsize=8) + fig.tight_layout() + p = os.path.join(out, f"stress_{axis}.png") + fig.savefig(p, dpi=110) + plt.close(fig) + made.append(p) + return made + + def main(): ap = argparse.ArgumentParser(description="Build E2E report_data.json") ap.add_argument("--repo", required=True) ap.add_argument("--results", required=True) ap.add_argument("--work", required=True) ap.add_argument("--out", required=True) + ap.add_argument("--stress-results", default=None) args = ap.parse_args() os.makedirs(args.out, exist_ok=True) @@ -250,12 +372,33 @@ def main(): cov = _load(os.path.join(args.out, "coverage.json"), {"languages": []}) e2e = build_e2e(results, args.work) + direct = build_by_kind(results, "direct") + recorder = build_by_kind(results, "recorder") + debug = build_by_kind(results, "debug") + tcplogger = build_by_kind(results, "tcplogger") + + stress = None + stress_plot_paths = [] + if args.stress_results and os.path.exists(args.stress_results): + sr = _load(args.stress_results) + if sr: + stress = build_stress(sr) + stress_plot_paths = stress_plots(stress["by_axis"], args.out) + now = datetime.datetime.now() meta = {"timestamp": now.isoformat(timespec="seconds"), "ts_short": now.strftime("%m-%d %H:%M"), "git_sha": _git_sha(args.repo), "target": "x86-linux"} hl = headline(e2e, ut, cov) + hl.update({ + "direct_pass": direct["n_pass"], "direct_fail": direct["n_fail"], + "recorder_pass": recorder["n_pass"], "recorder_fail": recorder["n_fail"], + "debug_pass": debug["n_pass"], "debug_fail": debug["n_fail"], + "tcplogger_pass": tcplogger["n_pass"], "tcplogger_fail": tcplogger["n_fail"], + }) + if stress: + hl.update(stress_headline(stress)) # history: read previous, then append current hist_path = os.path.join(args.out, "history.jsonl") @@ -283,10 +426,11 @@ def main(): plots = [os.path.basename(p) for p in trend_plots(history, args.out)] doc = { - "meta": meta, "e2e": e2e, "unit_tests": ut, - "coverage": cov, "regression": reg, "headline": hl, - "trend_plots": plots, "history_len": len(history), - "is_first_run": prev is None, + "meta": meta, "e2e": e2e, "unit_tests": ut, "coverage": cov, + "direct": direct, "recorder": recorder, "debug": debug, "tcplogger": tcplogger, + "stress": stress, "stress_plots": [os.path.basename(p) for p in stress_plot_paths], + "regression": reg, "headline": hl, "trend_plots": plots, + "history_len": len(history), "is_first_run": prev is None, } with open(os.path.join(args.out, "report_data.json"), "w") as f: json.dump(doc, f, indent=2) diff --git a/Test/E2E/suite/run_e2e.sh b/Test/E2E/suite/run_e2e.sh index 26c8c2c..f9e474f 100755 --- a/Test/E2E/suite/run_e2e.sh +++ b/Test/E2E/suite/run_e2e.sh @@ -479,7 +479,8 @@ fi # ── Consolidated report data (+ history/regression + trend plots) ──────────── ${PY} "${SCRIPT_DIR}/report_build.py" --repo "${REPO_ROOT}" \ - --results "${OUT_DIR}/results.json" --work "${WORK}" --out "${OUT_DIR}" || true + --results "${OUT_DIR}/results.json" --work "${WORK}" --out "${OUT_DIR}" \ + --stress-results "${OUT_DIR}/stress/stress_results.json" || true # ── PDF ────────────────────────────────────────────────────────────────────── if command -v typst >/dev/null 2>&1 && [ -f "${SCRIPT_DIR}/E2E_Report.typ" ]; then diff --git a/Test/E2E/suite/tests_py.py b/Test/E2E/suite/tests_py.py index 65af44a..fd1103f 100644 --- a/Test/E2E/suite/tests_py.py +++ b/Test/E2E/suite/tests_py.py @@ -85,6 +85,18 @@ class TestScenarios(unittest.TestCase): tcplogger_ids = {s["id"] for s in S.SCENARIOS if s["kind"] == "tcplogger"} self.assertEqual(tcplogger_ids, {"s56_tcplogger_delivery"}) + def test_build_by_kind_filters_correctly(self): + import report_build as R + results = {"scenarios": [ + {"id": "a", "kind": "direct", "status": "PASS"}, + {"id": "b", "kind": "chain", "status": "PASS"}, + {"id": "c", "kind": "direct", "status": "FAIL"}, + ]} + d = R.build_by_kind(results, "direct") + self.assertEqual(d["n_pass"], 1) + self.assertEqual(d["n_fail"], 1) + self.assertEqual(d["n_total"], 2) + class TestGenData(unittest.TestCase): def test_ground_truth_shapes(self):