martino/MARTe-Integrated-Components
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(e2e): render direct/recorder/debug/tcplogger/stress sections in the unified report

Browse Source 
Extends report_build.py with build_by_kind() (per-scenario-kind pass/fail
rollup) and a ported build_stress()/stress_headline()/stress_plots() (scaling
curves per stress axis), wires both into the headline KPIs, regression
tracking, and report_data.json. E2E_Report.typ renders the four new
per-kind tables plus a Stress Tests section (per-axis case tables + scaling
plots, gracefully degrading to placeholders when a kind/stress data is
absent). run_e2e.sh now passes --stress-results so the report actually
receives real stress data instead of silently omitting it.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Martino Ferrari
2026-07-01 21:57:11 +02:00
parent b65ac06ce2
commit 8337d678be
4 changed files with 235 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Test/E2E/suite/report_build.py
+148 -4
View File
@@ -139,6 +139,23 @@ def build_e2e(results, work):
}
def build_by_kind(results, kind):
"""Filter raw scenario records (as written by run_e2e.sh's results.json
aggregation) down to a single scenario `kind` (direct/recorder/debug/
tcplogger), with a small pass/fail rollup for the report's headline KPIs
and per-kind section."""
scenarios = [s for s in results.get("scenarios", []) if s.get("kind") == kind]
n_pass = sum(1 for s in scenarios if s.get("status") == "PASS")
n_fail = sum(1 for s in scenarios if s.get("status") == "FAIL")
return {
"kind": kind,
"n_pass": n_pass,
"n_fail": n_fail,
"n_total": len(scenarios),
"scenarios": scenarios,
}
def headline(e2e, ut, cov):
cov_by = {c["name"]: c.get("pct") for c in cov.get("languages", [])}
t = ut.get("totals", {})
@@ -236,12 +253,117 @@ def trend_plots(history, out):
return made
# ── stress (Test/E2E/suite/stress.py + stress_run.py) ───────────────────────
_STRESS_LABELS = {
"stress_pass": "Stress cases passed",
"stress_fail": "Stress cases failed",
"stress_max_hub_rss_mb": "Stress max hub RSS (MB)",
"stress_max_marte_rss_mb": "Stress max MARTe RSS (MB)",
"stress_max_zoom_p95_ms": "Stress max zoom p95 (ms)",
}
_STRESS_DIRECTION = {
"stress_pass": True, "stress_fail": False,
"stress_max_hub_rss_mb": False, "stress_max_marte_rss_mb": False,
"stress_max_zoom_p95_ms": False,
}
_DIRECTION.update({
"direct_pass": True, "direct_fail": False,
"recorder_pass": True, "recorder_fail": False,
"debug_pass": True, "debug_fail": False,
"tcplogger_pass": True, "tcplogger_fail": False,
})
_DIRECTION.update(_STRESS_DIRECTION)
_LABELS.update({
"direct_pass": "Direct scenarios passed", "direct_fail": "Direct scenarios failed",
"recorder_pass": "Recorder scenarios passed", "recorder_fail": "Recorder scenarios failed",
"debug_pass": "Debug scenarios passed", "debug_fail": "Debug scenarios failed",
"tcplogger_pass": "TCPLogger scenarios passed", "tcplogger_fail": "TCPLogger scenarios failed",
})
_LABELS.update(_STRESS_LABELS)
def build_stress(sr):
"""Shape stress_results.json into the report's stress block (+ by_axis)."""
cases = sr.get("cases", []) or []
by_axis = {}
for c in cases:
by_axis.setdefault(c.get("axis", "?"), []).append(c)
for axis in by_axis:
by_axis[axis].sort(key=lambda c: c.get("level", 0))
return {"overall": sr.get("overall", "FAIL"), "cases": cases,
"by_axis": by_axis}
def stress_headline(stress):
cases = stress.get("cases", []) or []
return {
"stress_pass": sum(1 for c in cases if c.get("status") == "PASS"),
"stress_fail": sum(1 for c in cases if c.get("status") == "FAIL"),
"stress_max_hub_rss_mb": max((c.get("hub_rss_mb", 0) or 0
for c in cases), default=0.0),
"stress_max_marte_rss_mb": max((c.get("marte_rss_mb", 0) or 0
for c in cases), default=0.0),
"stress_max_zoom_p95_ms": max((c.get("zoom_p95_ms", 0) or 0
for c in cases), default=0.0),
}
_STRESS_AXIS_METRICS = {
"ds_signal_elements": [("MARTe RSS (MB)", "marte_rss_mb"),
("hub RSS (MB)", "hub_rss_mb")],
"hub_signal_elements": [("hub RSS (MB)", "hub_rss_mb"),
("hub CPU (s)", "hub_cpu_s")],
"ds_signal_count": [("MARTe RSS (MB)", "marte_rss_mb"),
("MARTe CPU (s)", "marte_cpu_s")],
"hub_source_count": [("hub RSS (MB)", "hub_rss_mb"),
("MARTe RSS (MB)", "marte_rss_mb")],
"hub_ws_clients": [("hub RSS (MB)", "hub_rss_mb"),
("hub CPU (s)", "hub_cpu_s")],
"ds_subscriber_hubs": [("hub RSS (MB)", "hub_rss_mb"),
("MARTe CPU (s)", "marte_cpu_s")],
"hub_zoom_reqrate_hz": [("zoom p95 (ms)", "zoom_p95_ms"),
("zoom p50 (ms)", "zoom_p50_ms")],
}
def stress_plots(by_axis, out):
"""One scaling-curve PNG per axis: level (x) vs the axis's metrics (y)."""
made = []
for axis, cases in by_axis.items():
series = _STRESS_AXIS_METRICS.get(
axis, [("hub RSS (MB)", "hub_rss_mb"), ("MARTe RSS (MB)", "marte_rss_mb")])
xs = [c.get("level") for c in cases]
fig, ax = plt.subplots(figsize=(7, 3))
plotted = False
for lbl, field in series:
ys = [c.get(field) for c in cases]
if all((v is None or v == 0) for v in ys):
continue
ax.plot(xs, ys, "o-", label=lbl)
plotted = True
if not plotted:
plt.close(fig)
continue
ax.set_title(f"Scaling: {axis}")
ax.set_xlabel("load level")
ax.set_ylabel("value")
ax.grid(alpha=0.3)
ax.legend(fontsize=8)
fig.tight_layout()
p = os.path.join(out, f"stress_{axis}.png")
fig.savefig(p, dpi=110)
plt.close(fig)
made.append(p)
return made
def main():
ap = argparse.ArgumentParser(description="Build E2E report_data.json")
ap.add_argument("--repo", required=True)
ap.add_argument("--results", required=True)
ap.add_argument("--work", required=True)
ap.add_argument("--out", required=True)
ap.add_argument("--stress-results", default=None)
args = ap.parse_args()
os.makedirs(args.out, exist_ok=True)
@@ -250,12 +372,33 @@ def main():
cov = _load(os.path.join(args.out, "coverage.json"), {"languages": []})
e2e = build_e2e(results, args.work)
direct = build_by_kind(results, "direct")
recorder = build_by_kind(results, "recorder")
debug = build_by_kind(results, "debug")
tcplogger = build_by_kind(results, "tcplogger")
stress = None
stress_plot_paths = []
if args.stress_results and os.path.exists(args.stress_results):
sr = _load(args.stress_results)
if sr:
stress = build_stress(sr)
stress_plot_paths = stress_plots(stress["by_axis"], args.out)
now = datetime.datetime.now()
meta = {"timestamp": now.isoformat(timespec="seconds"),
"ts_short": now.strftime("%m-%d %H:%M"),
"git_sha": _git_sha(args.repo), "target": "x86-linux"}
hl = headline(e2e, ut, cov)
hl.update({
"direct_pass": direct["n_pass"], "direct_fail": direct["n_fail"],
"recorder_pass": recorder["n_pass"], "recorder_fail": recorder["n_fail"],
"debug_pass": debug["n_pass"], "debug_fail": debug["n_fail"],
"tcplogger_pass": tcplogger["n_pass"], "tcplogger_fail": tcplogger["n_fail"],
})
if stress:
hl.update(stress_headline(stress))
# history: read previous, then append current
hist_path = os.path.join(args.out, "history.jsonl")
@@ -283,10 +426,11 @@ def main():
plots = [os.path.basename(p) for p in trend_plots(history, args.out)]
doc = {
"meta": meta, "e2e": e2e, "unit_tests": ut,
"coverage": cov, "regression": reg, "headline": hl,
"trend_plots": plots, "history_len": len(history),
"is_first_run": prev is None,
"meta": meta, "e2e": e2e, "unit_tests": ut, "coverage": cov,
"direct": direct, "recorder": recorder, "debug": debug, "tcplogger": tcplogger,
"stress": stress, "stress_plots": [os.path.basename(p) for p in stress_plot_paths],
"regression": reg, "headline": hl, "trend_plots": plots,
"history_len": len(history), "is_first_run": prev is None,
}
with open(os.path.join(args.out, "report_data.json"), "w") as f:
json.dump(doc, f, indent=2)