refactor(e2e): rename Test/E2E/chain -> Test/E2E/suite, run_chain_e2e.sh -> run_e2e.sh
This commit is contained in:
Martino Ferrari
@@ -0,0 +1,332 @@
|
||||
// E2E_Report.typ — Streaming-chain E2E report.
|
||||
//
|
||||
// Renders report_data.json (produced by report_build.py) into a single PDF:
|
||||
// headline KPIs, per-field progression/regression vs the previous run, unit-test
|
||||
// suites, code coverage, performance (CPU/peak-RSS/throughput), per-scenario
|
||||
// waveform fidelity, trend plots, and embedded per-scenario waveform images.
|
||||
//
|
||||
// Compile from the directory holding report_data.json + the *.png artifacts:
|
||||
// typst compile E2E_Report.typ E2E_Report.pdf
|
||||
// Missing inputs degrade to "n/a" / skipped sections so a partial run still
|
||||
// renders.
|
||||
|
||||
#let data = json("report_data.json")
|
||||
#let meta = data.meta
|
||||
#let e2e = data.e2e
|
||||
#let ut = data.unit_tests
|
||||
#let cov = data.coverage
|
||||
#let reg = data.regression
|
||||
#let hl = data.headline
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────
|
||||
#let fmt(v, suffix: "") = if v == none { "n/a" } else { str(v) + suffix }
|
||||
#let fnum(v, digits: 3, suffix: "") = {
|
||||
if v == none { "n/a" } else {
|
||||
[#calc.round(v, digits: digits)] + suffix
|
||||
}
|
||||
}
|
||||
#let ok_color = rgb("#1a7f37")
|
||||
#let bad_color = rgb("#cf222e")
|
||||
#let neutral = rgb("#57606a")
|
||||
|
||||
#let warn_color = rgb("#9a6700") // XFAIL — expected/known failure
|
||||
#let xpass_color = rgb("#8250df") // XPASS — stale marker, needs attention
|
||||
#let status_badge(s) = {
|
||||
let c = if s == "PASS" { ok_color }
|
||||
else if s == "FAIL" { bad_color }
|
||||
else if s == "XFAIL" { warn_color }
|
||||
else if s == "XPASS" { xpass_color }
|
||||
else { neutral }
|
||||
box(fill: c, inset: (x: 6pt, y: 2pt), radius: 3pt, text(fill: white, weight: "bold", size: 8pt)[#s])
|
||||
}
|
||||
|
||||
#let yesno(b) = {
|
||||
if b == true { text(fill: ok_color)[✓] }
|
||||
else if b == false { text(fill: bad_color)[✗] }
|
||||
else { text(fill: neutral)[—] }
|
||||
}
|
||||
|
||||
// progression arrow for a regression row
|
||||
#let trend_arrow(r) = {
|
||||
if r.delta == none or r.delta == 0 { text(fill: neutral)[—] }
|
||||
else {
|
||||
let up = r.delta > 0
|
||||
let good = r.better == true
|
||||
let c = if good { ok_color } else { bad_color }
|
||||
let arrow = if up { "▲" } else { "▼" }
|
||||
text(fill: c)[#arrow #calc.round(r.delta, digits: 4)]
|
||||
}
|
||||
}
|
||||
|
||||
// ── page setup ───────────────────────────────────────────────────────────────
|
||||
#set page(
|
||||
paper: "a4", margin: (x: 1.8cm, y: 1.8cm),
|
||||
header: align(right, text(size: 8pt, fill: neutral)[Streaming-chain E2E · #meta.git_sha]),
|
||||
footer: context align(center, text(size: 8pt, fill: neutral)[#counter(page).display("1 / 1", both: true)]),
|
||||
)
|
||||
#set text(font: "DejaVu Sans", size: 9.5pt)
|
||||
#set heading(numbering: "1.1")
|
||||
#show heading.where(level: 1): it => block(above: 14pt, below: 8pt, text(size: 14pt, weight: "bold", it))
|
||||
#show heading.where(level: 2): it => block(above: 10pt, below: 6pt, text(size: 11pt, weight: "bold", it))
|
||||
|
||||
// ── title ────────────────────────────────────────────────────────────────────
|
||||
#align(center)[
|
||||
#text(size: 20pt, weight: "bold")[MARTe2 Streaming-Chain E2E Report] \
|
||||
#v(2pt)
|
||||
#text(size: 10pt, fill: neutral)[
|
||||
#meta.timestamp · commit #raw(meta.git_sha) · target #meta.target ·
|
||||
run \##data.history_len
|
||||
]
|
||||
]
|
||||
|
||||
#v(6pt)
|
||||
#align(center, status_badge(e2e.overall) + h(8pt) + text(size: 11pt)[
|
||||
E2E #hl.e2e_pass/#hl.e2e_total · Units #hl.unit_pass/#hl.unit_total
|
||||
#if hl.at("e2e_xfail", default: 0) > 0 [ · #hl.e2e_xfail xfail]
|
||||
#if hl.at("e2e_xpass", default: 0) > 0 [ · #text(fill: xpass_color)[#hl.e2e_xpass xpass!]]
|
||||
])
|
||||
|
||||
// ── headline KPIs ────────────────────────────────────────────────────────────
|
||||
#v(10pt)
|
||||
#let kpi(label, value) = box(width: 100%, inset: 8pt, radius: 4pt, fill: rgb("#f6f8fa"))[
|
||||
#align(center)[
|
||||
#text(size: 14pt, weight: "bold")[#value] \
|
||||
#text(size: 8pt, fill: neutral)[#label]
|
||||
]
|
||||
]
|
||||
#grid(columns: 4, gutter: 6pt,
|
||||
kpi("E2E passed", [#hl.e2e_pass/#hl.e2e_total]),
|
||||
kpi("Unit passed", [#hl.unit_pass/#hl.unit_total]),
|
||||
kpi("Mean sine corr", fnum(hl.mean_corr, digits: 4)),
|
||||
kpi("Throughput", fnum(hl.mean_throughput_sps, digits: 0, suffix: " sp/s")),
|
||||
kpi("Python cov", fmt(hl.cov_python, suffix: "%")),
|
||||
kpi("Go cov", fmt(hl.cov_go, suffix: "%")),
|
||||
kpi("Mean peak RSS", fnum(hl.mean_peak_rss_mb, digits: 1, suffix: " MB")),
|
||||
kpi("Mean CPU", fnum(hl.mean_cpu_s, digits: 2, suffix: " s")),
|
||||
)
|
||||
|
||||
// ── progression / regression ─────────────────────────────────────────────────
|
||||
= Progression / regression
|
||||
#if data.is_first_run [
|
||||
_First recorded run — baseline established; no previous run to compare against._
|
||||
] else [
|
||||
Comparison against the previous run (#text(fill: ok_color)[▲] better,
|
||||
#text(fill: bad_color)[▼] worse, — unchanged/unavailable).
|
||||
#v(4pt)
|
||||
#table(
|
||||
columns: (1.6fr, 1fr, 1fr, 1.2fr),
|
||||
align: (left, right, right, right),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 5pt,
|
||||
table.header([*Metric*], [*Current*], [*Previous*], [*Δ (trend)*]),
|
||||
..reg.map(r => (
|
||||
[#r.name],
|
||||
fnum(r.current, digits: 4),
|
||||
fnum(r.previous, digits: 4),
|
||||
trend_arrow(r),
|
||||
)).flatten()
|
||||
)
|
||||
]
|
||||
|
||||
// ── unit tests ───────────────────────────────────────────────────────────────
|
||||
= Unit tests
|
||||
#table(
|
||||
columns: (2fr, 0.8fr, 0.8fr, 0.8fr, 0.8fr, 0.8fr, 0.8fr),
|
||||
align: (left, center, right, right, right, right, right),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 5pt,
|
||||
table.header([*Suite*], [*Status*], [*Total*], [*Pass*], [*Fail*], [*Skip*], [*Time (s)*]),
|
||||
..ut.suites.map(s => (
|
||||
[#s.name],
|
||||
if not s.avail { status_badge("SKIP") } else if s.ok { status_badge("PASS") } else { status_badge("FAIL") },
|
||||
[#s.total], [#s.passed], [#s.failed], [#s.skipped], fnum(s.time_s, digits: 2),
|
||||
)).flatten(),
|
||||
table.cell(colspan: 2)[*Totals*],
|
||||
[#ut.totals.total], [#ut.totals.passed], [#ut.totals.failed], [#ut.totals.skipped], [],
|
||||
)
|
||||
|
||||
// ── coverage ─────────────────────────────────────────────────────────────────
|
||||
= Code coverage
|
||||
#table(
|
||||
columns: (1fr, 1fr, 2fr),
|
||||
align: (left, right, left),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 5pt,
|
||||
table.header([*Language*], [*Coverage*], [*Source*]),
|
||||
..cov.languages.map(l => (
|
||||
[#l.name],
|
||||
if l.pct == none { text(fill: neutral)[n/a] } else { [#l.pct%] },
|
||||
text(size: 8pt, fill: neutral)[#l.note],
|
||||
)).flatten()
|
||||
)
|
||||
|
||||
// per-file C++ detail — the streaming chain's most critical code, so the report
|
||||
// shows line coverage for every project C++ source (worst-covered first).
|
||||
#let cpp = cov.languages.find(l => l.name == "C++")
|
||||
#if cpp != none and cpp.at("files", default: ()).len() > 0 [
|
||||
== C++ source detail
|
||||
#let cf = cpp.files
|
||||
#text(size: 8pt, fill: neutral)[
|
||||
#cpp.at("lines_hit", default: 0) of #cpp.at("lines_found", default: 0)
|
||||
lines covered across #cf.len() project files (worst-covered first;
|
||||
green ≥75%, amber 50–75%, red under 50%).
|
||||
]
|
||||
#v(4pt)
|
||||
#let covcell(p) = {
|
||||
let c = if p == none { neutral }
|
||||
else if p >= 75 { ok_color }
|
||||
else if p >= 50 { warn_color }
|
||||
else { bad_color }
|
||||
text(fill: c, weight: "bold")[#if p == none { "n/a" } else [#p%]]
|
||||
}
|
||||
#table(
|
||||
columns: (3.2fr, 1fr, 1fr, 1.1fr),
|
||||
align: (left, right, right, right),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 4pt,
|
||||
table.header([*Source file*], [*Lines*], [*Hit*], [*Coverage*]),
|
||||
..cf.map(f => (
|
||||
text(size: 8pt)[#f.path],
|
||||
[#f.lines_found],
|
||||
[#f.lines_hit],
|
||||
covcell(f.pct),
|
||||
)).flatten()
|
||||
)
|
||||
]
|
||||
|
||||
// ── performance ──────────────────────────────────────────────────────────────
|
||||
= Performance
|
||||
Per-scenario CPU time and peak resident memory (VmHWM) of the StreamHub and
|
||||
MARTe2 processes, plus sustained client throughput (recorded samples ÷ duration).
|
||||
#v(4pt)
|
||||
#table(
|
||||
columns: (1.8fr, 1fr, 1fr, 1fr, 1fr, 1.1fr),
|
||||
align: (left, right, right, right, right, right),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 5pt,
|
||||
table.header([*Scenario*], [*Hub CPU (s)*], [*Hub RSS (MB)*],
|
||||
[*MARTe CPU (s)*], [*MARTe RSS (MB)*], [*Throughput (sp/s)*]),
|
||||
..e2e.scenarios.map(sc => {
|
||||
let h = sc.perf.at("hub", default: (:))
|
||||
let m = sc.perf.at("marte", default: (:))
|
||||
(
|
||||
[#sc.id],
|
||||
fnum(h.at("cpu_s", default: none), digits: 2),
|
||||
fnum(h.at("peak_rss_mb", default: none), digits: 1),
|
||||
fnum(m.at("cpu_s", default: none), digits: 2),
|
||||
fnum(m.at("peak_rss_mb", default: none), digits: 1),
|
||||
fnum(sc.throughput_sps, digits: 0),
|
||||
)
|
||||
}).flatten()
|
||||
)
|
||||
|
||||
// ── per-scenario waveform fidelity ───────────────────────────────────────────
|
||||
= Scenarios
|
||||
#for sc in e2e.scenarios [
|
||||
== #sc.id #h(6pt) #status_badge(sc.status)
|
||||
#if sc.at("desc", default: none) != none [
|
||||
#v(1pt)
|
||||
#text(size: 9pt, fill: neutral, style: "italic")[#sc.desc]
|
||||
#v(2pt)
|
||||
]
|
||||
#if sc.at("known_issue", default: none) != none [
|
||||
#v(2pt)
|
||||
#box(fill: rgb("#fff8c5"), inset: 5pt, radius: 3pt, width: 100%,
|
||||
text(size: 8pt, fill: rgb("#7d4e00"))[*Known issue:* #sc.known_issue])
|
||||
]
|
||||
#let rb = sc.rollup
|
||||
Client checks:
|
||||
live #yesno(rb.at("live_ok", default: none)) ·
|
||||
zoom #yesno(rb.at("zoom_ok", default: none)) ·
|
||||
window #yesno(rb.at("window_ok", default: none)) ·
|
||||
trigger #yesno(rb.at("trigger_ok", default: none))
|
||||
#if sc.live_frames != none [ · #sc.live_frames live frames]
|
||||
#v(3pt)
|
||||
#table(
|
||||
columns: (1.6fr, 0.6fr, 0.8fr, 0.8fr, 1fr, 0.9fr, 0.9fr, 0.8fr, 0.8fr),
|
||||
align: (left, center, left, left, right, right, right, center, center),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 4pt,
|
||||
table.header([*Signal*], [*Pass*], [*Type*], [*Quant*], [*Max abs err*],
|
||||
[*Corr*], [*nRMSE*], [*Fidelity*], [*Shape*]),
|
||||
..sc.signals.map(g => (
|
||||
raw(g.key),
|
||||
yesno(g.pass),
|
||||
text(size: 8pt)[#g.type],
|
||||
text(size: 8pt)[#g.quant],
|
||||
fnum(g.max_abs_err, digits: 6),
|
||||
fnum(g.corr, digits: 4),
|
||||
fnum(g.nrmse, digits: 4),
|
||||
yesno(g.fidelity_ok),
|
||||
yesno(g.shape_ok),
|
||||
)).flatten()
|
||||
)
|
||||
// zoom / window / trigger behavioural detail (real WS-driven results)
|
||||
#let zooms = sc.at("zoom", default: ())
|
||||
#let win = sc.at("window", default: (:))
|
||||
#let trigs = sc.at("trigger", default: ())
|
||||
#if zooms.len() > 0 or win.len() > 0 [
|
||||
#v(3pt)
|
||||
#text(size: 8.5pt, weight: "bold")[Zoom / window queries]
|
||||
#table(
|
||||
columns: (1fr, 1.2fr, 1fr, 0.8fr, 0.8fr),
|
||||
align: (left, left, right, right, center),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 4pt,
|
||||
table.header([*Query*], [*Signal*], [*Span (s)*], [*Pts*], [*OK*]),
|
||||
..zooms.enumerate().map(((i, z)) => {
|
||||
let r = z.at("range", default: (0, 0))
|
||||
(
|
||||
[zoom #str(i + 1)],
|
||||
raw(z.at("key", default: "")),
|
||||
fnum(r.at(1) - r.at(0), digits: 4),
|
||||
[#z.at("returned", default: 0)],
|
||||
yesno(z.at("inrange", default: none)),
|
||||
)
|
||||
}).flatten(),
|
||||
..(if win.len() > 0 and win.at("returned", default: 0) > 0 {(
|
||||
[window #fnum(win.at("windowSec", default: 0), digits: 2)],
|
||||
raw(win.at("key", default: "")),
|
||||
fnum(win.at("span", default: 0), digits: 4),
|
||||
[#win.at("returned", default: 0)],
|
||||
yesno(win.at("ok", default: none)),
|
||||
)} else {()}),
|
||||
)
|
||||
]
|
||||
#if trigs.len() > 0 [
|
||||
#v(3pt)
|
||||
#text(size: 8.5pt, weight: "bold")[Trigger captures] #h(4pt)
|
||||
#text(size: 8pt, fill: neutral)[(signal #raw(trigs.at(0).at("key", default: "")))]
|
||||
#table(
|
||||
columns: (1fr, 1fr, 0.8fr, 1fr, 1fr, 0.8fr, 0.8fr),
|
||||
align: (left, left, center, right, right, center, center),
|
||||
stroke: 0.4pt + rgb("#d0d7de"),
|
||||
inset: 4pt,
|
||||
table.header([*Edge*], [*Mode*], [*Fired*], [*Pre/Post (s)*],
|
||||
[*Capture pts*], [*Win OK*], [*Edge OK*]),
|
||||
..trigs.map(t => (
|
||||
text(size: 8pt)[#t.at("edge", default: "")],
|
||||
text(size: 8pt)[#t.at("mode", default: "")],
|
||||
yesno(t.at("fired", default: none)),
|
||||
text(size: 8pt)[#fnum(t.at("preSec", default: 0), digits: 3) / #fnum(t.at("postSec", default: 0), digits: 3)],
|
||||
[#t.at("capturePts", default: 0)],
|
||||
yesno(t.at("windowOk", default: none)),
|
||||
yesno(t.at("edgeOk", default: none)),
|
||||
)).flatten()
|
||||
)
|
||||
]
|
||||
// embed the waveform overview if report_build.py found one in the artifacts
|
||||
#if sc.at("wave_img", default: none) != none [
|
||||
#v(3pt)
|
||||
#align(center, image(sc.wave_img, width: 92%))
|
||||
]
|
||||
#v(6pt)
|
||||
]
|
||||
|
||||
// ── trend plots ──────────────────────────────────────────────────────────────
|
||||
#if data.trend_plots.len() > 0 [
|
||||
= Trends over runs
|
||||
#grid(columns: 2, gutter: 8pt,
|
||||
..data.trend_plots.map(p => image(p, width: 100%))
|
||||
)
|
||||
]
|
||||
Reference in New Issue
Block a user