diff --git a/Test/E2E/chain/report_build.py b/Test/E2E/chain/report_build.py index 21917ba..a029831 100644 --- a/Test/E2E/chain/report_build.py +++ b/Test/E2E/chain/report_build.py @@ -174,9 +174,11 @@ _LABELS = { } -def regression(curr, prev): +def regression(curr, prev, labels=None, directions=None): + labels = labels if labels is not None else _LABELS + directions = directions if directions is not None else _DIRECTION rows = [] - for k, label in _LABELS.items(): + for k, label in labels.items(): c = curr.get(k) p = prev.get(k) if prev else None better = None @@ -186,13 +188,105 @@ def regression(curr, prev): if delta == 0: better = None else: - better = (delta > 0) == _DIRECTION[k] + better = (delta > 0) == directions[k] rows.append({"name": label, "key": k, "current": c, "previous": p, "delta": delta, "better": better, - "higher_better": _DIRECTION[k]}) + "higher_better": directions[k]}) return rows +# Stress-axis aggregate metrics tracked across runs (mirrors the headline scalars). +_STRESS_LABELS = { + "stress_pass": "Stress cases passed", + "stress_fail": "Stress cases failed", + "stress_max_hub_rss_mb": "Stress max hub RSS (MB)", + "stress_max_marte_rss_mb": "Stress max MARTe RSS (MB)", + "stress_max_zoom_p95_ms": "Stress max zoom p95 (ms)", +} +_STRESS_DIRECTION = { + "stress_pass": True, "stress_fail": False, + "stress_max_hub_rss_mb": False, "stress_max_marte_rss_mb": False, + "stress_max_zoom_p95_ms": False, +} + + +def build_stress(sr): + """Shape stress_results.json into the report's stress block (+ by_axis).""" + cases = sr.get("cases", []) or [] + by_axis = {} + for c in cases: + by_axis.setdefault(c.get("axis", "?"), []).append(c) + for axis in by_axis: + by_axis[axis].sort(key=lambda c: c.get("level", 0)) + return {"overall": sr.get("overall", "FAIL"), "cases": cases, + "by_axis": by_axis} + + +def stress_headline(stress): + cases = stress.get("cases", []) or [] + return { + "stress_pass": sum(1 for c in cases if c.get("status") == "PASS"), + "stress_fail": sum(1 for c in cases if c.get("status") == "FAIL"), + "stress_max_hub_rss_mb": max((c.get("hub_rss_mb", 0) or 0 + for c in cases), default=0.0), + "stress_max_marte_rss_mb": max((c.get("marte_rss_mb", 0) or 0 + for c in cases), default=0.0), + "stress_max_zoom_p95_ms": max((c.get("zoom_p95_ms", 0) or 0 + for c in cases), default=0.0), + } + + +# Which metrics to plot per axis (label, case-field). Mixed units share a "value" +# y-axis as trend_perf.png already does; all-zero series are dropped. +_STRESS_AXIS_METRICS = { + "ds_signal_elements": [("MARTe RSS (MB)", "marte_rss_mb"), + ("hub RSS (MB)", "hub_rss_mb")], + "hub_signal_elements": [("hub RSS (MB)", "hub_rss_mb"), + ("hub CPU (s)", "hub_cpu_s")], + "ds_signal_count": [("MARTe RSS (MB)", "marte_rss_mb"), + ("MARTe CPU (s)", "marte_cpu_s")], + "hub_source_count": [("hub RSS (MB)", "hub_rss_mb"), + ("MARTe RSS (MB)", "marte_rss_mb")], + "hub_ws_clients": [("hub RSS (MB)", "hub_rss_mb"), + ("hub CPU (s)", "hub_cpu_s")], + "ds_subscriber_hubs": [("hub RSS (MB)", "hub_rss_mb"), + ("MARTe CPU (s)", "marte_cpu_s")], + "hub_zoom_reqrate_hz": [("zoom p95 (ms)", "zoom_p95_ms"), + ("zoom p50 (ms)", "zoom_p50_ms")], +} + + +def stress_plots(by_axis, out): + """One scaling-curve PNG per axis: level (x) vs the axis's metrics (y).""" + made = [] + for axis, cases in by_axis.items(): + series = _STRESS_AXIS_METRICS.get( + axis, [("hub RSS (MB)", "hub_rss_mb"), ("MARTe RSS (MB)", "marte_rss_mb")]) + xs = [c.get("level") for c in cases] + fig, ax = plt.subplots(figsize=(7, 3)) + plotted = False + for lbl, field in series: + ys = [c.get(field) for c in cases] + if all((v is None or v == 0) for v in ys): + continue + ax.plot(xs, ys, "o-", label=lbl) + plotted = True + if not plotted: + plt.close(fig) + continue + ax.set_title(f"Scaling: {axis}") + ax.set_xlabel("load level") + ax.set_ylabel("value") + ax.grid(alpha=0.3) + ax.legend(fontsize=8) + fig.tight_layout() + p = os.path.join(out, f"stress_{axis}.png") + fig.savefig(p, dpi=110) + plt.close(fig) + made.append(p) + return made + + def trend_plots(history, out): if not history: return [] @@ -242,12 +336,16 @@ def main(): ap.add_argument("--results", required=True) ap.add_argument("--work", required=True) ap.add_argument("--out", required=True) + ap.add_argument("--stress-results", default="", + help="path to stress_results.json (optional)") args = ap.parse_args() os.makedirs(args.out, exist_ok=True) results = _load(args.results, {"overall": "FAIL", "scenarios": []}) ut = _load(os.path.join(args.out, "unit_tests.json"), {"suites": [], "totals": {}}) cov = _load(os.path.join(args.out, "coverage.json"), {"languages": []}) + sr = _load(args.stress_results) if args.stress_results else None + stress = build_stress(sr) if sr else None e2e = build_e2e(results, args.work) now = datetime.datetime.now() @@ -256,6 +354,11 @@ def main(): "git_sha": _git_sha(args.repo), "target": "x86-linux"} hl = headline(e2e, ut, cov) + labels, directions = _LABELS, _DIRECTION + if stress: + hl.update(stress_headline(stress)) + labels = {**_LABELS, **_STRESS_LABELS} + directions = {**_DIRECTION, **_STRESS_DIRECTION} # history: read previous, then append current hist_path = os.path.join(args.out, "history.jsonl") @@ -269,24 +372,35 @@ def main(): except ValueError: pass prev = history[-1] if history else None - reg = regression(hl, prev) + reg = regression(hl, prev, labels, directions) entry = dict(hl) entry["timestamp"] = meta["timestamp"] entry["ts_short"] = meta["ts_short"] entry["git_sha"] = meta["git_sha"] entry["overall"] = e2e["overall"] + if stress: + entry["stress"] = [ + {k: c.get(k) for k in ("id", "axis", "level", "status", + "marte_cpu_s", "marte_rss_mb", + "hub_cpu_s", "hub_rss_mb", + "zoom_p95_ms", "min_frames")} + for c in stress["cases"] + ] with open(hist_path, "a") as f: f.write(json.dumps(entry) + "\n") history.append(entry) plots = [os.path.basename(p) for p in trend_plots(history, args.out)] + splots = ([os.path.basename(p) for p in stress_plots(stress["by_axis"], args.out)] + if stress else []) doc = { "meta": meta, "e2e": e2e, "unit_tests": ut, "coverage": cov, "regression": reg, "headline": hl, "trend_plots": plots, "history_len": len(history), "is_first_run": prev is None, + "stress": stress, "stress_plots": splots, } with open(os.path.join(args.out, "report_data.json"), "w") as f: json.dump(doc, f, indent=2) diff --git a/Test/E2E/chain/test_report_stress.py b/Test/E2E/chain/test_report_stress.py new file mode 100644 index 0000000..6049875 --- /dev/null +++ b/Test/E2E/chain/test_report_stress.py @@ -0,0 +1,63 @@ +import os +import report_build as RB + +_SR = { + "overall": "PASS", + "cases": [ + {"id": "ds_size_1000", "shape": "hub", "axis": "ds_signal_elements", + "level": 1000, "status": "PASS", "survival": True, "clients": 1, + "min_frames": 200, "marte_cpu_s": 12.8, "marte_rss_mb": 10.4, + "hub_cpu_s": 2.33, "hub_rss_mb": 28.3, "zoom_count": 0, "zoom_fail": 0, + "zoom_p50_ms": 0.0, "zoom_p95_ms": 0.0, "fails": []}, + {"id": "ds_size_4000", "shape": "hub", "axis": "ds_signal_elements", + "level": 4000, "status": "PASS", "survival": True, "clients": 1, + "min_frames": 180, "marte_cpu_s": 20.0, "marte_rss_mb": 14.0, + "hub_cpu_s": 3.0, "hub_rss_mb": 40.0, "zoom_count": 0, "zoom_fail": 0, + "zoom_p50_ms": 0.0, "zoom_p95_ms": 0.0, "fails": []}, + {"id": "hub_reqrate_50", "shape": "hub", "axis": "hub_zoom_reqrate_hz", + "level": 50, "status": "PASS", "survival": True, "clients": 4, + "min_frames": 100, "marte_cpu_s": 5.0, "marte_rss_mb": 12.0, + "hub_cpu_s": 8.0, "hub_rss_mb": 60.0, "zoom_count": 400, "zoom_fail": 0, + "zoom_p50_ms": 12.0, "zoom_p95_ms": 35.0, "fails": []}, + ], +} + + +def test_build_stress_groups_by_axis_sorted_by_level(): + st = RB.build_stress(_SR) + assert st["overall"] == "PASS" + assert len(st["cases"]) == 3 + assert set(st["by_axis"]) == {"ds_signal_elements", "hub_zoom_reqrate_hz"} + levels = [c["level"] for c in st["by_axis"]["ds_signal_elements"]] + assert levels == [1000, 4000] # sorted ascending + + +def test_stress_headline_aggregates(): + st = RB.build_stress(_SR) + hl = RB.stress_headline(st) + assert hl["stress_pass"] == 3 + assert hl["stress_fail"] == 0 + assert hl["stress_max_hub_rss_mb"] == 60.0 + assert hl["stress_max_marte_rss_mb"] == 14.0 + assert hl["stress_max_zoom_p95_ms"] == 35.0 + + +def test_stress_plots_one_png_per_axis(tmp_path): + st = RB.build_stress(_SR) + made = RB.stress_plots(st["by_axis"], str(tmp_path)) + names = {os.path.basename(p) for p in made} + assert "stress_ds_signal_elements.png" in names + assert "stress_hub_zoom_reqrate_hz.png" in names + for p in made: + assert os.path.exists(p) + + +def test_regression_includes_stress_when_present(): + curr = {"e2e_pass": 5, "stress_max_hub_rss_mb": 60.0} + prev = {"e2e_pass": 5, "stress_max_hub_rss_mb": 50.0} + labels = dict(RB._LABELS); labels["stress_max_hub_rss_mb"] = "Stress max hub RSS (MB)" + directions = dict(RB._DIRECTION); directions["stress_max_hub_rss_mb"] = False + rows = RB.regression(curr, prev, labels, directions) + row = next(r for r in rows if r["key"] == "stress_max_hub_rss_mb") + assert row["delta"] == 10.0 + assert row["better"] is False # RSS went up → worse