feat(e2e): render direct/recorder/debug/tcplogger/stress sections in the unified report
Extends report_build.py with build_by_kind() (per-scenario-kind pass/fail rollup) and a ported build_stress()/stress_headline()/stress_plots() (scaling curves per stress axis), wires both into the headline KPIs, regression tracking, and report_data.json. E2E_Report.typ renders the four new per-kind tables plus a Stress Tests section (per-axis case tables + scaling plots, gracefully degrading to placeholders when a kind/stress data is absent). run_e2e.sh now passes --stress-results so the report actually receives real stress data instead of silently omitting it. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Martino Ferrari
@@ -323,6 +323,79 @@ MARTe2 processes, plus sustained client throughput (recorded samples ÷ duration
|
||||
#v(6pt)
|
||||
]
|
||||
|
||||
// ── per-kind sections (direct/recorder/debug/tcplogger) ─────────────────────
|
||||
// Raw scenario records here come straight from results.json (not e2e.scenarios'
|
||||
// reshaping), so they only carry id/kind/status/known_issue/metrics — no
|
||||
// waveform-fidelity breakdown (already covered, where applicable, in the
|
||||
// Scenarios section above for chain-kind scenarios; these kinds run through
|
||||
// dedicated non-chain harnesses in run_e2e.sh).
|
||||
#let kind_table(title, block) = {
|
||||
[= #title]
|
||||
[#block.n_pass/#block.n_total passed.]
|
||||
v(4pt)
|
||||
if block.n_total == 0 {
|
||||
text(fill: neutral)[_No scenarios of this kind in this run._]
|
||||
} else {
|
||||
table(
|
||||
columns: (1.4fr, 0.8fr, 2fr),
|
||||
align: (left, center, left),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 5pt,
|
||||
table.header([*Scenario*], [*Status*], [*Known issue*]),
|
||||
..block.scenarios.map(s => (
|
||||
[#s.id],
|
||||
status_badge(s.status),
|
||||
if s.at("known_issue", default: none) != none {
|
||||
text(size: 8pt, fill: rgb("#7d4e00"))[#s.known_issue]
|
||||
} else { text(fill: neutral)[—] },
|
||||
)).flatten()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#kind_table("Direct Round-Trip (UDPStreamer ↔ UDPStreamerClient)", data.direct)
|
||||
#kind_table("Recorder (BinaryRecorder disk output)", data.recorder)
|
||||
#kind_table("Debug Service E2E", data.debug)
|
||||
#kind_table("TCPLogger E2E", data.tcplogger)
|
||||
|
||||
// ── stress tests ─────────────────────────────────────────────────────────────
|
||||
= Stress Tests
|
||||
#if data.stress == none [
|
||||
_Not run this session._
|
||||
] else [
|
||||
#align(center, status_badge(data.stress.overall) + h(8pt) + text(size: 11pt)[
|
||||
#hl.at("stress_pass", default: 0) passed ·
|
||||
#hl.at("stress_fail", default: 0) failed
|
||||
])
|
||||
#v(4pt)
|
||||
#for (axis, cases) in data.stress.by_axis [
|
||||
== Axis: #raw(axis)
|
||||
#table(
|
||||
columns: (0.8fr, 0.8fr, 1fr, 1fr, 1fr, 1fr),
|
||||
align: (right, center, right, right, right, right),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 4pt,
|
||||
table.header([*Level*], [*Status*], [*Hub RSS (MB)*], [*MARTe RSS (MB)*],
|
||||
[*Zoom p50 (ms)*], [*Zoom p95 (ms)*]),
|
||||
..cases.map(c => (
|
||||
[#c.at("level", default: "n/a")],
|
||||
status_badge(c.at("status", default: "?")),
|
||||
fnum(c.at("hub_rss_mb", default: none), digits: 1),
|
||||
fnum(c.at("marte_rss_mb", default: none), digits: 1),
|
||||
fnum(c.at("zoom_p50_ms", default: none), digits: 1),
|
||||
fnum(c.at("zoom_p95_ms", default: none), digits: 1),
|
||||
)).flatten()
|
||||
)
|
||||
]
|
||||
#if data.stress_plots.len() > 0 [
|
||||
#v(4pt)
|
||||
== Scaling curves
|
||||
#grid(columns: 2, gutter: 8pt,
|
||||
..data.stress_plots.map(p => image(p, width: 100%))
|
||||
)
|
||||
]
|
||||
]
|
||||
|
||||
// ── trend plots ──────────────────────────────────────────────────────────────
|
||||
#if data.trend_plots.len() > 0 [
|
||||
= Trends over runs
|
||||
|
||||
@@ -139,6 +139,23 @@ def build_e2e(results, work):
|
||||
}
|
||||
|
||||
|
||||
def build_by_kind(results, kind):
|
||||
"""Filter raw scenario records (as written by run_e2e.sh's results.json
|
||||
aggregation) down to a single scenario `kind` (direct/recorder/debug/
|
||||
tcplogger), with a small pass/fail rollup for the report's headline KPIs
|
||||
and per-kind section."""
|
||||
scenarios = [s for s in results.get("scenarios", []) if s.get("kind") == kind]
|
||||
n_pass = sum(1 for s in scenarios if s.get("status") == "PASS")
|
||||
n_fail = sum(1 for s in scenarios if s.get("status") == "FAIL")
|
||||
return {
|
||||
"kind": kind,
|
||||
"n_pass": n_pass,
|
||||
"n_fail": n_fail,
|
||||
"n_total": len(scenarios),
|
||||
"scenarios": scenarios,
|
||||
}
|
||||
|
||||
|
||||
def headline(e2e, ut, cov):
|
||||
cov_by = {c["name"]: c.get("pct") for c in cov.get("languages", [])}
|
||||
t = ut.get("totals", {})
|
||||
@@ -236,12 +253,117 @@ def trend_plots(history, out):
|
||||
return made
|
||||
|
||||
|
||||
# ── stress (Test/E2E/suite/stress.py + stress_run.py) ───────────────────────
|
||||
_STRESS_LABELS = {
|
||||
"stress_pass": "Stress cases passed",
|
||||
"stress_fail": "Stress cases failed",
|
||||
"stress_max_hub_rss_mb": "Stress max hub RSS (MB)",
|
||||
"stress_max_marte_rss_mb": "Stress max MARTe RSS (MB)",
|
||||
"stress_max_zoom_p95_ms": "Stress max zoom p95 (ms)",
|
||||
}
|
||||
_STRESS_DIRECTION = {
|
||||
"stress_pass": True, "stress_fail": False,
|
||||
"stress_max_hub_rss_mb": False, "stress_max_marte_rss_mb": False,
|
||||
"stress_max_zoom_p95_ms": False,
|
||||
}
|
||||
_DIRECTION.update({
|
||||
"direct_pass": True, "direct_fail": False,
|
||||
"recorder_pass": True, "recorder_fail": False,
|
||||
"debug_pass": True, "debug_fail": False,
|
||||
"tcplogger_pass": True, "tcplogger_fail": False,
|
||||
})
|
||||
_DIRECTION.update(_STRESS_DIRECTION)
|
||||
_LABELS.update({
|
||||
"direct_pass": "Direct scenarios passed", "direct_fail": "Direct scenarios failed",
|
||||
"recorder_pass": "Recorder scenarios passed", "recorder_fail": "Recorder scenarios failed",
|
||||
"debug_pass": "Debug scenarios passed", "debug_fail": "Debug scenarios failed",
|
||||
"tcplogger_pass": "TCPLogger scenarios passed", "tcplogger_fail": "TCPLogger scenarios failed",
|
||||
})
|
||||
_LABELS.update(_STRESS_LABELS)
|
||||
|
||||
|
||||
def build_stress(sr):
|
||||
"""Shape stress_results.json into the report's stress block (+ by_axis)."""
|
||||
cases = sr.get("cases", []) or []
|
||||
by_axis = {}
|
||||
for c in cases:
|
||||
by_axis.setdefault(c.get("axis", "?"), []).append(c)
|
||||
for axis in by_axis:
|
||||
by_axis[axis].sort(key=lambda c: c.get("level", 0))
|
||||
return {"overall": sr.get("overall", "FAIL"), "cases": cases,
|
||||
"by_axis": by_axis}
|
||||
|
||||
|
||||
def stress_headline(stress):
|
||||
cases = stress.get("cases", []) or []
|
||||
return {
|
||||
"stress_pass": sum(1 for c in cases if c.get("status") == "PASS"),
|
||||
"stress_fail": sum(1 for c in cases if c.get("status") == "FAIL"),
|
||||
"stress_max_hub_rss_mb": max((c.get("hub_rss_mb", 0) or 0
|
||||
for c in cases), default=0.0),
|
||||
"stress_max_marte_rss_mb": max((c.get("marte_rss_mb", 0) or 0
|
||||
for c in cases), default=0.0),
|
||||
"stress_max_zoom_p95_ms": max((c.get("zoom_p95_ms", 0) or 0
|
||||
for c in cases), default=0.0),
|
||||
}
|
||||
|
||||
|
||||
_STRESS_AXIS_METRICS = {
|
||||
"ds_signal_elements": [("MARTe RSS (MB)", "marte_rss_mb"),
|
||||
("hub RSS (MB)", "hub_rss_mb")],
|
||||
"hub_signal_elements": [("hub RSS (MB)", "hub_rss_mb"),
|
||||
("hub CPU (s)", "hub_cpu_s")],
|
||||
"ds_signal_count": [("MARTe RSS (MB)", "marte_rss_mb"),
|
||||
("MARTe CPU (s)", "marte_cpu_s")],
|
||||
"hub_source_count": [("hub RSS (MB)", "hub_rss_mb"),
|
||||
("MARTe RSS (MB)", "marte_rss_mb")],
|
||||
"hub_ws_clients": [("hub RSS (MB)", "hub_rss_mb"),
|
||||
("hub CPU (s)", "hub_cpu_s")],
|
||||
"ds_subscriber_hubs": [("hub RSS (MB)", "hub_rss_mb"),
|
||||
("MARTe CPU (s)", "marte_cpu_s")],
|
||||
"hub_zoom_reqrate_hz": [("zoom p95 (ms)", "zoom_p95_ms"),
|
||||
("zoom p50 (ms)", "zoom_p50_ms")],
|
||||
}
|
||||
|
||||
|
||||
def stress_plots(by_axis, out):
|
||||
"""One scaling-curve PNG per axis: level (x) vs the axis's metrics (y)."""
|
||||
made = []
|
||||
for axis, cases in by_axis.items():
|
||||
series = _STRESS_AXIS_METRICS.get(
|
||||
axis, [("hub RSS (MB)", "hub_rss_mb"), ("MARTe RSS (MB)", "marte_rss_mb")])
|
||||
xs = [c.get("level") for c in cases]
|
||||
fig, ax = plt.subplots(figsize=(7, 3))
|
||||
plotted = False
|
||||
for lbl, field in series:
|
||||
ys = [c.get(field) for c in cases]
|
||||
if all((v is None or v == 0) for v in ys):
|
||||
continue
|
||||
ax.plot(xs, ys, "o-", label=lbl)
|
||||
plotted = True
|
||||
if not plotted:
|
||||
plt.close(fig)
|
||||
continue
|
||||
ax.set_title(f"Scaling: {axis}")
|
||||
ax.set_xlabel("load level")
|
||||
ax.set_ylabel("value")
|
||||
ax.grid(alpha=0.3)
|
||||
ax.legend(fontsize=8)
|
||||
fig.tight_layout()
|
||||
p = os.path.join(out, f"stress_{axis}.png")
|
||||
fig.savefig(p, dpi=110)
|
||||
plt.close(fig)
|
||||
made.append(p)
|
||||
return made
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Build E2E report_data.json")
|
||||
ap.add_argument("--repo", required=True)
|
||||
ap.add_argument("--results", required=True)
|
||||
ap.add_argument("--work", required=True)
|
||||
ap.add_argument("--out", required=True)
|
||||
ap.add_argument("--stress-results", default=None)
|
||||
args = ap.parse_args()
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
|
||||
@@ -250,12 +372,33 @@ def main():
|
||||
cov = _load(os.path.join(args.out, "coverage.json"), {"languages": []})
|
||||
|
||||
e2e = build_e2e(results, args.work)
|
||||
direct = build_by_kind(results, "direct")
|
||||
recorder = build_by_kind(results, "recorder")
|
||||
debug = build_by_kind(results, "debug")
|
||||
tcplogger = build_by_kind(results, "tcplogger")
|
||||
|
||||
stress = None
|
||||
stress_plot_paths = []
|
||||
if args.stress_results and os.path.exists(args.stress_results):
|
||||
sr = _load(args.stress_results)
|
||||
if sr:
|
||||
stress = build_stress(sr)
|
||||
stress_plot_paths = stress_plots(stress["by_axis"], args.out)
|
||||
|
||||
now = datetime.datetime.now()
|
||||
meta = {"timestamp": now.isoformat(timespec="seconds"),
|
||||
"ts_short": now.strftime("%m-%d %H:%M"),
|
||||
"git_sha": _git_sha(args.repo), "target": "x86-linux"}
|
||||
|
||||
hl = headline(e2e, ut, cov)
|
||||
hl.update({
|
||||
"direct_pass": direct["n_pass"], "direct_fail": direct["n_fail"],
|
||||
"recorder_pass": recorder["n_pass"], "recorder_fail": recorder["n_fail"],
|
||||
"debug_pass": debug["n_pass"], "debug_fail": debug["n_fail"],
|
||||
"tcplogger_pass": tcplogger["n_pass"], "tcplogger_fail": tcplogger["n_fail"],
|
||||
})
|
||||
if stress:
|
||||
hl.update(stress_headline(stress))
|
||||
|
||||
# history: read previous, then append current
|
||||
hist_path = os.path.join(args.out, "history.jsonl")
|
||||
@@ -283,10 +426,11 @@ def main():
|
||||
plots = [os.path.basename(p) for p in trend_plots(history, args.out)]
|
||||
|
||||
doc = {
|
||||
"meta": meta, "e2e": e2e, "unit_tests": ut,
|
||||
"coverage": cov, "regression": reg, "headline": hl,
|
||||
"trend_plots": plots, "history_len": len(history),
|
||||
"is_first_run": prev is None,
|
||||
"meta": meta, "e2e": e2e, "unit_tests": ut, "coverage": cov,
|
||||
"direct": direct, "recorder": recorder, "debug": debug, "tcplogger": tcplogger,
|
||||
"stress": stress, "stress_plots": [os.path.basename(p) for p in stress_plot_paths],
|
||||
"regression": reg, "headline": hl, "trend_plots": plots,
|
||||
"history_len": len(history), "is_first_run": prev is None,
|
||||
}
|
||||
with open(os.path.join(args.out, "report_data.json"), "w") as f:
|
||||
json.dump(doc, f, indent=2)
|
||||
|
||||
@@ -479,7 +479,8 @@ fi
|
||||
|
||||
# ── Consolidated report data (+ history/regression + trend plots) ────────────
|
||||
${PY} "${SCRIPT_DIR}/report_build.py" --repo "${REPO_ROOT}" \
|
||||
--results "${OUT_DIR}/results.json" --work "${WORK}" --out "${OUT_DIR}" || true
|
||||
--results "${OUT_DIR}/results.json" --work "${WORK}" --out "${OUT_DIR}" \
|
||||
--stress-results "${OUT_DIR}/stress/stress_results.json" || true
|
||||
|
||||
# ── PDF ──────────────────────────────────────────────────────────────────────
|
||||
if command -v typst >/dev/null 2>&1 && [ -f "${SCRIPT_DIR}/E2E_Report.typ" ]; then
|
||||
|
||||
@@ -85,6 +85,18 @@ class TestScenarios(unittest.TestCase):
|
||||
tcplogger_ids = {s["id"] for s in S.SCENARIOS if s["kind"] == "tcplogger"}
|
||||
self.assertEqual(tcplogger_ids, {"s56_tcplogger_delivery"})
|
||||
|
||||
def test_build_by_kind_filters_correctly(self):
|
||||
import report_build as R
|
||||
results = {"scenarios": [
|
||||
{"id": "a", "kind": "direct", "status": "PASS"},
|
||||
{"id": "b", "kind": "chain", "status": "PASS"},
|
||||
{"id": "c", "kind": "direct", "status": "FAIL"},
|
||||
]}
|
||||
d = R.build_by_kind(results, "direct")
|
||||
self.assertEqual(d["n_pass"], 1)
|
||||
self.assertEqual(d["n_fail"], 1)
|
||||
self.assertEqual(d["n_total"], 2)
|
||||
|
||||
|
||||
class TestGenData(unittest.TestCase):
|
||||
def test_ground_truth_shapes(self):
|
||||
|
||||
Reference in New Issue
Block a user