test(e2e-chain): fold stress results into report_data.json
report_build.py reads stress_results.json (when --stress-results given), adds a stress block (cases + by_axis), per-axis scaling-curve PNGs, aggregate stress headline metrics, and stress regression rows vs the previous run. Degrades to no stress section when the file is absent. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Martino Ferrari
@@ -174,9 +174,11 @@ _LABELS = {
|
||||
}
|
||||
|
||||
|
||||
def regression(curr, prev):
|
||||
def regression(curr, prev, labels=None, directions=None):
|
||||
labels = labels if labels is not None else _LABELS
|
||||
directions = directions if directions is not None else _DIRECTION
|
||||
rows = []
|
||||
for k, label in _LABELS.items():
|
||||
for k, label in labels.items():
|
||||
c = curr.get(k)
|
||||
p = prev.get(k) if prev else None
|
||||
better = None
|
||||
@@ -186,13 +188,105 @@ def regression(curr, prev):
|
||||
if delta == 0:
|
||||
better = None
|
||||
else:
|
||||
better = (delta > 0) == _DIRECTION[k]
|
||||
better = (delta > 0) == directions[k]
|
||||
rows.append({"name": label, "key": k, "current": c, "previous": p,
|
||||
"delta": delta, "better": better,
|
||||
"higher_better": _DIRECTION[k]})
|
||||
"higher_better": directions[k]})
|
||||
return rows
|
||||
|
||||
|
||||
# Stress-axis aggregate metrics tracked across runs (mirrors the headline scalars).
|
||||
_STRESS_LABELS = {
|
||||
"stress_pass": "Stress cases passed",
|
||||
"stress_fail": "Stress cases failed",
|
||||
"stress_max_hub_rss_mb": "Stress max hub RSS (MB)",
|
||||
"stress_max_marte_rss_mb": "Stress max MARTe RSS (MB)",
|
||||
"stress_max_zoom_p95_ms": "Stress max zoom p95 (ms)",
|
||||
}
|
||||
_STRESS_DIRECTION = {
|
||||
"stress_pass": True, "stress_fail": False,
|
||||
"stress_max_hub_rss_mb": False, "stress_max_marte_rss_mb": False,
|
||||
"stress_max_zoom_p95_ms": False,
|
||||
}
|
||||
|
||||
|
||||
def build_stress(sr):
|
||||
"""Shape stress_results.json into the report's stress block (+ by_axis)."""
|
||||
cases = sr.get("cases", []) or []
|
||||
by_axis = {}
|
||||
for c in cases:
|
||||
by_axis.setdefault(c.get("axis", "?"), []).append(c)
|
||||
for axis in by_axis:
|
||||
by_axis[axis].sort(key=lambda c: c.get("level", 0))
|
||||
return {"overall": sr.get("overall", "FAIL"), "cases": cases,
|
||||
"by_axis": by_axis}
|
||||
|
||||
|
||||
def stress_headline(stress):
|
||||
cases = stress.get("cases", []) or []
|
||||
return {
|
||||
"stress_pass": sum(1 for c in cases if c.get("status") == "PASS"),
|
||||
"stress_fail": sum(1 for c in cases if c.get("status") == "FAIL"),
|
||||
"stress_max_hub_rss_mb": max((c.get("hub_rss_mb", 0) or 0
|
||||
for c in cases), default=0.0),
|
||||
"stress_max_marte_rss_mb": max((c.get("marte_rss_mb", 0) or 0
|
||||
for c in cases), default=0.0),
|
||||
"stress_max_zoom_p95_ms": max((c.get("zoom_p95_ms", 0) or 0
|
||||
for c in cases), default=0.0),
|
||||
}
|
||||
|
||||
|
||||
# Which metrics to plot per axis (label, case-field). Mixed units share a "value"
|
||||
# y-axis as trend_perf.png already does; all-zero series are dropped.
|
||||
_STRESS_AXIS_METRICS = {
|
||||
"ds_signal_elements": [("MARTe RSS (MB)", "marte_rss_mb"),
|
||||
("hub RSS (MB)", "hub_rss_mb")],
|
||||
"hub_signal_elements": [("hub RSS (MB)", "hub_rss_mb"),
|
||||
("hub CPU (s)", "hub_cpu_s")],
|
||||
"ds_signal_count": [("MARTe RSS (MB)", "marte_rss_mb"),
|
||||
("MARTe CPU (s)", "marte_cpu_s")],
|
||||
"hub_source_count": [("hub RSS (MB)", "hub_rss_mb"),
|
||||
("MARTe RSS (MB)", "marte_rss_mb")],
|
||||
"hub_ws_clients": [("hub RSS (MB)", "hub_rss_mb"),
|
||||
("hub CPU (s)", "hub_cpu_s")],
|
||||
"ds_subscriber_hubs": [("hub RSS (MB)", "hub_rss_mb"),
|
||||
("MARTe CPU (s)", "marte_cpu_s")],
|
||||
"hub_zoom_reqrate_hz": [("zoom p95 (ms)", "zoom_p95_ms"),
|
||||
("zoom p50 (ms)", "zoom_p50_ms")],
|
||||
}
|
||||
|
||||
|
||||
def stress_plots(by_axis, out):
|
||||
"""One scaling-curve PNG per axis: level (x) vs the axis's metrics (y)."""
|
||||
made = []
|
||||
for axis, cases in by_axis.items():
|
||||
series = _STRESS_AXIS_METRICS.get(
|
||||
axis, [("hub RSS (MB)", "hub_rss_mb"), ("MARTe RSS (MB)", "marte_rss_mb")])
|
||||
xs = [c.get("level") for c in cases]
|
||||
fig, ax = plt.subplots(figsize=(7, 3))
|
||||
plotted = False
|
||||
for lbl, field in series:
|
||||
ys = [c.get(field) for c in cases]
|
||||
if all((v is None or v == 0) for v in ys):
|
||||
continue
|
||||
ax.plot(xs, ys, "o-", label=lbl)
|
||||
plotted = True
|
||||
if not plotted:
|
||||
plt.close(fig)
|
||||
continue
|
||||
ax.set_title(f"Scaling: {axis}")
|
||||
ax.set_xlabel("load level")
|
||||
ax.set_ylabel("value")
|
||||
ax.grid(alpha=0.3)
|
||||
ax.legend(fontsize=8)
|
||||
fig.tight_layout()
|
||||
p = os.path.join(out, f"stress_{axis}.png")
|
||||
fig.savefig(p, dpi=110)
|
||||
plt.close(fig)
|
||||
made.append(p)
|
||||
return made
|
||||
|
||||
|
||||
def trend_plots(history, out):
|
||||
if not history:
|
||||
return []
|
||||
@@ -242,12 +336,16 @@ def main():
|
||||
ap.add_argument("--results", required=True)
|
||||
ap.add_argument("--work", required=True)
|
||||
ap.add_argument("--out", required=True)
|
||||
ap.add_argument("--stress-results", default="",
|
||||
help="path to stress_results.json (optional)")
|
||||
args = ap.parse_args()
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
|
||||
results = _load(args.results, {"overall": "FAIL", "scenarios": []})
|
||||
ut = _load(os.path.join(args.out, "unit_tests.json"), {"suites": [], "totals": {}})
|
||||
cov = _load(os.path.join(args.out, "coverage.json"), {"languages": []})
|
||||
sr = _load(args.stress_results) if args.stress_results else None
|
||||
stress = build_stress(sr) if sr else None
|
||||
|
||||
e2e = build_e2e(results, args.work)
|
||||
now = datetime.datetime.now()
|
||||
@@ -256,6 +354,11 @@ def main():
|
||||
"git_sha": _git_sha(args.repo), "target": "x86-linux"}
|
||||
|
||||
hl = headline(e2e, ut, cov)
|
||||
labels, directions = _LABELS, _DIRECTION
|
||||
if stress:
|
||||
hl.update(stress_headline(stress))
|
||||
labels = {**_LABELS, **_STRESS_LABELS}
|
||||
directions = {**_DIRECTION, **_STRESS_DIRECTION}
|
||||
|
||||
# history: read previous, then append current
|
||||
hist_path = os.path.join(args.out, "history.jsonl")
|
||||
@@ -269,24 +372,35 @@ def main():
|
||||
except ValueError:
|
||||
pass
|
||||
prev = history[-1] if history else None
|
||||
reg = regression(hl, prev)
|
||||
reg = regression(hl, prev, labels, directions)
|
||||
|
||||
entry = dict(hl)
|
||||
entry["timestamp"] = meta["timestamp"]
|
||||
entry["ts_short"] = meta["ts_short"]
|
||||
entry["git_sha"] = meta["git_sha"]
|
||||
entry["overall"] = e2e["overall"]
|
||||
if stress:
|
||||
entry["stress"] = [
|
||||
{k: c.get(k) for k in ("id", "axis", "level", "status",
|
||||
"marte_cpu_s", "marte_rss_mb",
|
||||
"hub_cpu_s", "hub_rss_mb",
|
||||
"zoom_p95_ms", "min_frames")}
|
||||
for c in stress["cases"]
|
||||
]
|
||||
with open(hist_path, "a") as f:
|
||||
f.write(json.dumps(entry) + "\n")
|
||||
history.append(entry)
|
||||
|
||||
plots = [os.path.basename(p) for p in trend_plots(history, args.out)]
|
||||
splots = ([os.path.basename(p) for p in stress_plots(stress["by_axis"], args.out)]
|
||||
if stress else [])
|
||||
|
||||
doc = {
|
||||
"meta": meta, "e2e": e2e, "unit_tests": ut,
|
||||
"coverage": cov, "regression": reg, "headline": hl,
|
||||
"trend_plots": plots, "history_len": len(history),
|
||||
"is_first_run": prev is None,
|
||||
"stress": stress, "stress_plots": splots,
|
||||
}
|
||||
with open(os.path.join(args.out, "report_data.json"), "w") as f:
|
||||
json.dump(doc, f, indent=2)
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import os
|
||||
import report_build as RB
|
||||
|
||||
_SR = {
|
||||
"overall": "PASS",
|
||||
"cases": [
|
||||
{"id": "ds_size_1000", "shape": "hub", "axis": "ds_signal_elements",
|
||||
"level": 1000, "status": "PASS", "survival": True, "clients": 1,
|
||||
"min_frames": 200, "marte_cpu_s": 12.8, "marte_rss_mb": 10.4,
|
||||
"hub_cpu_s": 2.33, "hub_rss_mb": 28.3, "zoom_count": 0, "zoom_fail": 0,
|
||||
"zoom_p50_ms": 0.0, "zoom_p95_ms": 0.0, "fails": []},
|
||||
{"id": "ds_size_4000", "shape": "hub", "axis": "ds_signal_elements",
|
||||
"level": 4000, "status": "PASS", "survival": True, "clients": 1,
|
||||
"min_frames": 180, "marte_cpu_s": 20.0, "marte_rss_mb": 14.0,
|
||||
"hub_cpu_s": 3.0, "hub_rss_mb": 40.0, "zoom_count": 0, "zoom_fail": 0,
|
||||
"zoom_p50_ms": 0.0, "zoom_p95_ms": 0.0, "fails": []},
|
||||
{"id": "hub_reqrate_50", "shape": "hub", "axis": "hub_zoom_reqrate_hz",
|
||||
"level": 50, "status": "PASS", "survival": True, "clients": 4,
|
||||
"min_frames": 100, "marte_cpu_s": 5.0, "marte_rss_mb": 12.0,
|
||||
"hub_cpu_s": 8.0, "hub_rss_mb": 60.0, "zoom_count": 400, "zoom_fail": 0,
|
||||
"zoom_p50_ms": 12.0, "zoom_p95_ms": 35.0, "fails": []},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_build_stress_groups_by_axis_sorted_by_level():
|
||||
st = RB.build_stress(_SR)
|
||||
assert st["overall"] == "PASS"
|
||||
assert len(st["cases"]) == 3
|
||||
assert set(st["by_axis"]) == {"ds_signal_elements", "hub_zoom_reqrate_hz"}
|
||||
levels = [c["level"] for c in st["by_axis"]["ds_signal_elements"]]
|
||||
assert levels == [1000, 4000] # sorted ascending
|
||||
|
||||
|
||||
def test_stress_headline_aggregates():
|
||||
st = RB.build_stress(_SR)
|
||||
hl = RB.stress_headline(st)
|
||||
assert hl["stress_pass"] == 3
|
||||
assert hl["stress_fail"] == 0
|
||||
assert hl["stress_max_hub_rss_mb"] == 60.0
|
||||
assert hl["stress_max_marte_rss_mb"] == 14.0
|
||||
assert hl["stress_max_zoom_p95_ms"] == 35.0
|
||||
|
||||
|
||||
def test_stress_plots_one_png_per_axis(tmp_path):
|
||||
st = RB.build_stress(_SR)
|
||||
made = RB.stress_plots(st["by_axis"], str(tmp_path))
|
||||
names = {os.path.basename(p) for p in made}
|
||||
assert "stress_ds_signal_elements.png" in names
|
||||
assert "stress_hub_zoom_reqrate_hz.png" in names
|
||||
for p in made:
|
||||
assert os.path.exists(p)
|
||||
|
||||
|
||||
def test_regression_includes_stress_when_present():
|
||||
curr = {"e2e_pass": 5, "stress_max_hub_rss_mb": 60.0}
|
||||
prev = {"e2e_pass": 5, "stress_max_hub_rss_mb": 50.0}
|
||||
labels = dict(RB._LABELS); labels["stress_max_hub_rss_mb"] = "Stress max hub RSS (MB)"
|
||||
directions = dict(RB._DIRECTION); directions["stress_max_hub_rss_mb"] = False
|
||||
rows = RB.regression(curr, prev, labels, directions)
|
||||
row = next(r for r in rows if r["key"] == "stress_max_hub_rss_mb")
|
||||
assert row["delta"] == 10.0
|
||||
assert row["better"] is False # RSS went up → worse
|
||||
Reference in New Issue
Block a user