MARTe-Integrated-Components/Test/E2E/suite/run_e2e.sh

#!/usr/bin/env bash
# run_e2e.sh — Full-chain E2E orchestrator for the streaming chain
#
#   MARTe2 app (FileReader -> IOGAM -> UDPStreamer)
#     -> UDPS -> StreamHub -> chain-client (record + zoom/window/trigger)
#       -> validate_waveform.py -> plots.py -> results.json [-> PDF]
#
# Per scenario (scenarios.py) it generates input data + both cfgs, runs the
# two-process stack, drives the mock client, validates the recorded waveform
# against the analytic/fed oracle, renders plots, and aggregates results.json.
# Artifacts: Build/x86-linux/E2E/chain/ (report) and /tmp/chain_e2e/ (scratch).
#
# Usage: ./run_e2e.sh [--skip-build] [--only <id>] [--pdf-only] [--cpp-coverage]
#                      [--skip-coverage] [--skip-stress] [--skip-datasources]
#                      [--skip-recorder] [--skip-debug] [--skip-tcplogger]
set -u

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
TARGET=x86-linux
BUILD_DIR="${REPO_ROOT}/Build/${TARGET}"
OUT_DIR="${BUILD_DIR}/E2E/chain"
WORK="/tmp/chain_e2e"
mkdir -p "${OUT_DIR}" "${WORK}"

SKIP_BUILD=0
ONLY=""
PDF_ONLY=0
CPP_COV=1
SKIP_STRESS=0
SKIP_DATASOURCES=0
SKIP_RECORDER=0
SKIP_DEBUG=0
SKIP_TCPLOGGER=0
while [ $# -gt 0 ]; do
    case "$1" in
        --skip-build) SKIP_BUILD=1 ;;
        --only) shift; ONLY="$1" ;;
        --pdf-only) PDF_ONLY=1 ;;
        --cpp-coverage) CPP_COV=1 ;;
        --skip-coverage) CPP_COV=0 ;;
        --skip-stress) SKIP_STRESS=1 ;;
        --skip-datasources) SKIP_DATASOURCES=1 ;;
        --skip-recorder) SKIP_RECORDER=1 ;;
        --skip-debug) SKIP_DEBUG=1 ;;
        --skip-tcplogger) SKIP_TCPLOGGER=1 ;;
        --help|-h) echo "Usage: $0 [--skip-build] [--only <id>] [--pdf-only] [--cpp-coverage] [--skip-coverage] [--skip-stress] [--skip-datasources] [--skip-recorder] [--skip-debug] [--skip-tcplogger]"; exit 0 ;;
        *) echo "unknown arg $1" >&2; exit 2 ;;
    esac
    shift
done

ENV_SCRIPT="${REPO_ROOT}/env.sh"
[ -f "${ENV_SCRIPT}" ] || { echo "ERROR: ${ENV_SCRIPT} not found" >&2; exit 1; }
: "${LD_LIBRARY_PATH:=}"   # env.sh appends to it under our set -u
source "${ENV_SCRIPT}"

COMP="${MARTe2_Components_DIR}/Build/${TARGET}/Components"
export LD_LIBRARY_PATH="\
${BUILD_DIR}/Components/DataSources/UDPStreamer:\
${BUILD_DIR}/Components/DataSources/UDPStreamerClient:\
${BUILD_DIR}/Components/Interfaces/UDPStream:\
${BUILD_DIR}/Components/Interfaces/DebugService:\
${BUILD_DIR}/Components/Interfaces/TCPLogger:\
${MARTe2_DIR}/Build/${TARGET}/Core:\
${COMP}/DataSources/LinuxTimer:\
${COMP}/DataSources/FileDataSource:\
${COMP}/GAMs/IOGAM:\
${LD_LIBRARY_PATH:-}"

MARTE_APP="${MARTe2_DIR}/Build/${TARGET}/App/MARTeApp.ex"
STREAMHUB_EX="${BUILD_DIR}/StreamHub/StreamHub.ex"
CLIENT="${SCRIPT_DIR}/client/chain-client"
DEBUGCLIENT="${SCRIPT_DIR}/debugclient/debugclient"

PY="python3"
SCEN() { ${PY} "${SCRIPT_DIR}/scenarios.py" >/dev/null 2>&1; }

# ── PDF-only shortcut (Task 9 fills in the compile) ──────────────────────────
if [ "${PDF_ONLY}" -eq 1 ]; then
    if command -v typst >/dev/null 2>&1 && [ -f "${SCRIPT_DIR}/E2E_Report.typ" ]; then
        cp "${SCRIPT_DIR}/E2E_Report.typ" "${OUT_DIR}/" 2>/dev/null || true
        (cd "${OUT_DIR}" && typst compile E2E_Report.typ E2E_Report.pdf) \
            && echo "PDF: ${OUT_DIR}/E2E_Report.pdf"
    else
        echo "typst or E2E_Report.typ missing — skipping PDF"
    fi
    exit 0
fi

# ── Build ────────────────────────────────────────────────────────────────────
if [ "${SKIP_BUILD}" -eq 0 ]; then
    echo "── Building components ──"
    make -C "${REPO_ROOT}/Source/Components/Interfaces/UDPStream" -f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -1
    make -C "${REPO_ROOT}/Source/Components/DataSources/UDPStreamer" -f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -1
    make -C "${REPO_ROOT}/Source/Components/DataSources/UDPStreamerClient" -f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -1
    make -C "${REPO_ROOT}/Source/Applications/StreamHub" -f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -1
fi
if [ ! -x "${CLIENT}" ]; then
    echo "── Building chain-client ──"
    (cd "${SCRIPT_DIR}/client" && go build -o chain-client .) || { echo "client build failed"; exit 1; }
fi
if [ ! -x "${DEBUGCLIENT}" ]; then
    echo "── Building debugclient ──"
    (cd "${SCRIPT_DIR}/debugclient" && go build -o debugclient .) || { echo "debugclient build failed"; exit 1; }
fi
[ -x "${MARTE_APP}" ] || { echo "ERROR: MARTeApp.ex not found at ${MARTE_APP}" >&2; exit 1; }
[ -x "${STREAMHUB_EX}" ] || { echo "ERROR: StreamHub.ex not found" >&2; exit 1; }

# ── Scenario list (kind|id|f2|f3|f4|f5|f6|f7) ────────────────────────────────
# chain:            kind|id|ws_port|udp_port0|network|oracle|trig|checks
# direct:           kind|id|cfg|-|-|-|-|-
# recorder:         kind|id|marte_cfg|hub_cfg|-|-|-|-
# debug/tcplogger:  kind|id|cfg|cmd_port|udp_port|log_port|-|-
SKIP_KINDS=""
[ "${SKIP_DATASOURCES}" -eq 1 ] && SKIP_KINDS="${SKIP_KINDS}direct,"
[ "${SKIP_RECORDER}" -eq 1 ] && SKIP_KINDS="${SKIP_KINDS}recorder,"
[ "${SKIP_DEBUG}" -eq 1 ] && SKIP_KINDS="${SKIP_KINDS}debug,"
[ "${SKIP_TCPLOGGER}" -eq 1 ] && SKIP_KINDS="${SKIP_KINDS}tcplogger,"
LIST="$(${PY} - "${ONLY}" "${SKIP_KINDS}" <<'PY'
import sys, os
sys.path.insert(0, os.path.dirname(os.path.abspath("Test/E2E/suite/scenarios.py")))
sys.path.insert(0, os.path.join(os.getcwd(), "Test/E2E/suite"))
import scenarios as S
only = sys.argv[1] if len(sys.argv) > 1 else ""
skip_kinds = set(sys.argv[2].split(",")) if len(sys.argv) > 2 and sys.argv[2] else set()
for s in S.SCENARIOS:
    if only and s["id"] != only:
        continue
    if s["kind"] in skip_kinds:
        continue
    kind = s["kind"]
    if kind == "chain":
        trig = s.get("trig_signal") or ""
        checks = ",".join(s.get("client_checks", []))
        if not trig:
            checks = ",".join(c for c in s.get("client_checks", []) if c != "trigger")
        print("|".join([kind, s["id"], str(s["ws_port"]), str(s["sources"][0]["udp_port"]),
                        s["network"], s["oracle"], trig, checks]))
    elif kind == "direct":
        print("|".join([kind, s["id"], s["cfg"], "", "", "", "", ""]))
    elif kind == "recorder":
        print("|".join([kind, s["id"], s["marte_cfg"], s["hub_cfg"], "", "", "", ""]))
    elif kind == "debug":
        print("|".join([kind, s["id"], s["cfg"], str(s["cmd_port"]), str(s["udp_port"]), str(s["log_port"]), "", ""]))
    elif kind == "tcplogger":
        print("|".join([kind, s["id"], s["cfg"], str(s["cmd_port"]), str(s["udp_port"]), str(s["log_port"]), "", ""]))
PY
)"

if [ -z "${LIST}" ]; then echo "no scenarios selected"; exit 1; fi

# run_scenario_matrix — runs the full scenario matrix (already computed into
# ${LIST}) against whatever binaries are currently built. Invoked once for the
# authoritative pass (normal binaries) and again, inside a subshell with
# OUT_DIR overridden, for the coverage-only re-run against instrumented
# binaries (see Phase 4 below). Running the coverage pass in a subshell means
# its SCEN_IDS/OUT_DIR/HUB_PID/APP_PID/trap mutations never leak back into the
# parent shell, so the primary pass's SCEN_IDS (consumed by the results.json
# aggregation step further down) is unaffected regardless of call order.
run_scenario_matrix() {
    local RESULTS_SUFFIX="$1"
SCEN_IDS=""
HUB_PID=""; APP_PID=""
cleanup() {
    [ -n "${APP_PID}" ] && kill "${APP_PID}" 2>/dev/null
    [ -n "${HUB_PID}" ] && kill "${HUB_PID}" 2>/dev/null
    wait "${APP_PID}" 2>/dev/null; wait "${HUB_PID}" 2>/dev/null
    APP_PID=""; HUB_PID=""
}
trap cleanup EXIT

while IFS='|' read -r KIND ID F2 F3 F4 F5 F6 F7; do
    [ -z "${ID}" ] && continue
    SCEN_IDS="${SCEN_IDS} ${ID}"
    : > "${WORK}/status_${ID}.txt"

    case "${KIND}" in
    chain)
    WSPORT="${F2}"; UDPPORT="${F3}"; NET="${F4}"; ORACLE="${F5}"; TRIG="${F6}"; CHECKS="${F7}"
    echo ""
    echo "══ scenario ${ID}  (net=${NET} oracle=${ORACLE} ws=${WSPORT}) ══"

    # multicast route probe
    if [ "${NET}" = "multicast" ]; then
        GRP="$(${PY} -c "import sys;sys.path.insert(0,'${SCRIPT_DIR}');import scenarios as S;print(next(s for s in S.SCENARIOS if s['id']=='${ID}')['sources'][0]['multicast_group'])")"
        if ! ip route get "${GRP}" >/dev/null 2>&1; then
            echo "  SKIP: no multicast route to ${GRP}"
            echo "SKIP" > "${WORK}/status_${ID}.txt"
            continue
        fi
    fi

    INPUT="${WORK}/input_${ID}.bin"
    MCFG="${WORK}/m_${ID}.cfg"
    HCFG="${WORK}/h_${ID}.cfg"
    TAP=""
    if [ "${ORACLE}" = "fed" ] || [ "${ORACLE}" = "both" ]; then
        TAP="${WORK}/tap_${ID}.bin"
    fi

    ${PY} "${SCRIPT_DIR}/gen_data.py" --scenario "${ID}" --out "${INPUT}" || { echo FAIL > "${WORK}/status_${ID}.txt"; continue; }
    if [ -n "${TAP}" ]; then
        ${PY} "${SCRIPT_DIR}/gen_cfg.py" --scenario "${ID}" --input "${INPUT}" --marte-out "${MCFG}" --hub-out "${HCFG}" --tap "${TAP}"
    else
        ${PY} "${SCRIPT_DIR}/gen_cfg.py" --scenario "${ID}" --input "${INPUT}" --marte-out "${MCFG}" --hub-out "${HCFG}"
    fi

    HUB_LOG="${OUT_DIR}/hub_${ID}.log"
    APP_LOG="${OUT_DIR}/marte_${ID}.log"
    rm -f "${WORK}/received_${ID}.bin" "${WORK}/checks_${ID}.json" "${WORK}/metrics_${ID}.json"

    "${STREAMHUB_EX}" -cfg "${HCFG}" > "${HUB_LOG}" 2>&1 &
    HUB_PID=$!
    sleep 1
    timeout 120 "${MARTE_APP}" -l RealTimeLoader -f "${MCFG}" -s Running > "${APP_LOG}" 2>&1 &
    APP_PID=$!
    sleep 1
    # APP_PID is the `timeout` wrapper; perf must target the real MARTeApp child.
    APP_PERF_PID="$(pgrep -P "${APP_PID}" 2>/dev/null | head -1)"
    [ -z "${APP_PERF_PID}" ] && APP_PERF_PID="${APP_PID}"

    TRIGARG=""
    [ -n "${TRIG}" ] && TRIGARG="-trigsig ${TRIG}"
    if "${CLIENT}" -hub "127.0.0.1:${WSPORT}" -scenario "${ID}" ${TRIGARG} \
            -checks "${CHECKS}" -out "${WORK}" -dur 4 > "${OUT_DIR}/client_${ID}.log" 2>&1; then
        echo "  client OK"
    else
        echo "  client FAILED (see client_${ID}.log)"
        tail -3 "${OUT_DIR}/client_${ID}.log" | sed 's/^/    /'
    fi

    # Snapshot CPU/peak-RSS while the stack is still alive (peak RSS is only
    # legible before exit), then tear it down.
    [ -n "${HUB_PID}" ] && ${PY} "${SCRIPT_DIR}/proc_perf.py" "${HUB_PID}" streamhub "${WORK}/perf_${ID}_hub.json" || true
    [ -n "${APP_PERF_PID}" ] && ${PY} "${SCRIPT_DIR}/proc_perf.py" "${APP_PERF_PID}" marte "${WORK}/perf_${ID}_marte.json" || true
    cleanup

    # validate + plot
    VARGS="--scenario ${ID} --received ${WORK}/received_${ID}.bin --checks ${WORK}/checks_${ID}.json --out ${WORK}/metrics_${ID}.json"
    [ -n "${TAP}" ] && [ -f "${TAP}" ] && VARGS="${VARGS} --tap ${TAP}"
    if ${PY} "${SCRIPT_DIR}/validate_waveform.py" ${VARGS}; then
        echo "PASS" > "${WORK}/status_${ID}.txt"
    else
        echo "FAIL" > "${WORK}/status_${ID}.txt"
    fi
    ${PY} "${SCRIPT_DIR}/plots.py" --scenario "${ID}" --dir "${WORK}" >/dev/null 2>&1 || true
    ;;

    direct)
    CFG="${F2}"
    echo ""
    echo "══ scenario ${ID}  (kind=direct cfg=${CFG}) ══"

    DS_DIR="${SCRIPT_DIR}/../datasources"
    INPUT_BIN="/tmp/udpstreamer_test_input.bin"
    # The FileWriterDS output path is baked into the cfg (not per-scenario);
    # the last `Filename = "..."` in the file is always the writer (the
    # FileReaderDS input path is always listed first in these cfgs).
    OUTPUT_BIN="$(grep -oP 'Filename\s*=\s*"\K[^"]+' "${REPO_ROOT}/${CFG}" | tail -1)"
    rm -f "${INPUT_BIN}" "${OUTPUT_BIN}"
    (cd "${DS_DIR}" && ${PY} -c "import gen_test_data; gen_test_data.generate()") > "${OUT_DIR}/gen_${ID}.log" 2>&1

    APP_LOG="${OUT_DIR}/marte_${ID}.log"
    timeout 8 "${MARTE_APP}" -l RealTimeLoader -f "${REPO_ROOT}/${CFG}" -s Running > "${APP_LOG}" 2>&1 &
    APP_PID=$!
    sleep 5
    cleanup

    MSG="$(${PY} -c "
import sys
sys.path.insert(0, '${DS_DIR}')
import validate_binary as V
ok, msg, details = V.validate('${INPUT_BIN}', '${OUTPUT_BIN}', label='${ID}')
print(msg)
sys.exit(0 if ok else 1)
" 2>&1)" && STATUS=PASS || STATUS=FAIL
    echo "  ${MSG}"
    echo "${STATUS}" > "${WORK}/status_${ID}.txt"
    echo "${ID}: ${STATUS}"
    ;;

    recorder)
    RCFG_MARTE="${F2}"; RCFG_HUB="${F3}"
    echo ""
    echo "══ scenario ${ID}  (kind=recorder marte_cfg=${RCFG_MARTE} hub_cfg=${RCFG_HUB}) ══"

    DS_DIR="${SCRIPT_DIR}/../datasources"
    INPUT_BIN="/tmp/udpstreamer_test_input.bin"
    REC_DIR="/tmp/streamhub_rec_e2e"
    rm -rf "${REC_DIR}"; mkdir -p "${REC_DIR}"
    (cd "${DS_DIR}" && ${PY} -c "import gen_test_data; gen_test_data.generate()") > "${OUT_DIR}/gen_${ID}.log" 2>&1

    HUB_LOG="${OUT_DIR}/hub_${ID}.log"
    APP_LOG="${OUT_DIR}/marte_${ID}.log"

    # Start StreamHub first so it is ready to receive the CONFIG packet.
    "${STREAMHUB_EX}" -cfg "${REPO_ROOT}/${RCFG_HUB}" > "${HUB_LOG}" 2>&1 &
    HUB_PID=$!
    sleep 1
    timeout 8 "${MARTE_APP}" -l RealTimeLoader -f "${REPO_ROOT}/${RCFG_MARTE}" -s Running > "${APP_LOG}" 2>&1 &
    APP_PID=$!

    # Let data flow, then stop the streamer and give the push/flush thread
    # time to write the recorder file before killing the hub.
    sleep 7
    kill "${APP_PID}" 2>/dev/null; wait "${APP_PID}" 2>/dev/null; APP_PID=""
    sleep 2
    cleanup

    # The recorder names files <sourceId>_<UTCstamp>_<seq>.bin; pick the newest.
    REC_FILE="$(ls -t "${REC_DIR}"/*.bin 2>/dev/null | head -1)"
    if [ -z "${REC_FILE}" ]; then
        echo "  FAIL: no recorder output file found in ${REC_DIR}"
        echo "FAIL" > "${WORK}/status_${ID}.txt"
        echo "${ID}: FAIL"
    else
        MSG="$(${PY} -c "
import sys
sys.path.insert(0, '${DS_DIR}')
import validate_binary as V
ok, msg, details = V.validate('${INPUT_BIN}', '${REC_FILE}', label='${ID}')
print(msg)
sys.exit(0 if ok else 1)
" 2>&1)" && STATUS=PASS || STATUS=FAIL
        echo "  ${MSG}"
        echo "${STATUS}" > "${WORK}/status_${ID}.txt"
        echo "${ID}: ${STATUS}"
    fi
    ;;

    debug|tcplogger)
    CFG="${F2}"; CMDPORT="${F3}"; UDPPORT2="${F4}"; LOGPORT="${F5}"
    echo ""
    echo "══ scenario ${ID}  (kind=${KIND} cfg=${CFG}) ══"
    timeout 15 "${MARTE_APP}" -l RealTimeLoader -f "${REPO_ROOT}/${CFG}" -s Running \
        > "${OUT_DIR}/marte_${ID}.log" 2>&1 &
    APP_PID=$!
    sleep 1
    "${DEBUGCLIENT}" -host 127.0.0.1 \
        -cmdport "${CMDPORT}" -udpport "${UDPPORT2}" -logport "${LOGPORT}" \
        -mode "${KIND}" -scenario "${ID}" -out "${WORK}" -dur 4s \
        > "${OUT_DIR}/client_${ID}.log" 2>&1
    CLIENT_RC=$?
    kill "${APP_PID}" 2>/dev/null; wait "${APP_PID}" 2>/dev/null; APP_PID=""
    if [ "${CLIENT_RC}" -eq 0 ]; then
        echo "PASS" > "${WORK}/status_${ID}.txt"
    else
        echo "FAIL" > "${WORK}/status_${ID}.txt"
    fi
    echo "${ID}: $(cat "${WORK}/status_${ID}.txt")"
    ;;

    *)
    echo "  SKIP: unknown scenario kind '${KIND}' for ${ID}"
    echo "SKIP" > "${WORK}/status_${ID}.txt"
    ;;
    esac
done <<< "${LIST}"

trap - EXIT
cleanup
}
run_scenario_matrix "primary"

# ── Stress (Phase 3: normal binaries, always uninstrumented) ────────────────
if [ "${SKIP_STRESS}" -eq 0 ]; then
    echo ""
    echo "── Stress matrix ──"
    STRESS_OUT="${OUT_DIR}/stress"
    mkdir -p "${STRESS_OUT}"
    ${PY} "${SCRIPT_DIR}/stress_run.py" --marte "${MARTE_APP}" --hub "${STREAMHUB_EX}" \
        --client "${CLIENT}" --work "/tmp/chain_stress" --out "${STRESS_OUT}" || true
fi

# ── Aggregate results.json ───────────────────────────────────────────────────
# Scenarios carrying a `known_issue` marker exercise a documented, not-yet-fixed
# chain gap: a raw FAIL is reclassified XFAIL (expected failure — does not break
# the green baseline) and a raw PASS becomes XPASS (the bug was unexpectedly
# fixed; the marker should be dropped). Overall is PASS when nothing FAILs and
# nothing unexpectedly XPASSes.
WORK="${WORK}" OUT_DIR="${OUT_DIR}" SCEN_IDS="${SCEN_IDS}" \
SCRIPT_DIR="${SCRIPT_DIR}" ${PY} - <<'PY'
import json, os, sys
work = os.environ["WORK"]; out = os.environ["OUT_DIR"]
ids = os.environ["SCEN_IDS"].split()
sys.path.insert(0, os.environ["SCRIPT_DIR"])
try:
    from scenarios import SCENARIOS
    known = {s["id"]: s.get("known_issue") for s in SCENARIOS}
    kind_map = {s["id"]: s["kind"] for s in SCENARIOS}
except Exception:
    known = {}
    kind_map = {}
results = []
for sid in ids:
    rec = {"id": sid, "kind": kind_map.get(sid, "chain")}
    st = os.path.join(work, f"status_{sid}.txt")
    raw = open(st).read().strip() if os.path.exists(st) else "UNKNOWN"
    ki = known.get(sid)
    if ki:
        rec["known_issue"] = ki
        if raw == "FAIL":
            raw = "XFAIL"          # expected failure — documented chain gap
        elif raw == "PASS":
            raw = "XPASS"          # unexpectedly fixed — drop the marker
    rec["status"] = raw
    mp = os.path.join(work, f"metrics_{sid}.json")
    if os.path.exists(mp):
        rec["metrics"] = json.load(open(mp))
    results.append(rec)
# Green when no hard FAIL and no XPASS (an XPASS means a known_issue marker is
# now stale and must be removed — surfaced as a failure to force the cleanup).
overall = (bool(results)
           and all(r["status"] in ("PASS", "SKIP", "XFAIL") for r in results))
doc = {"overall": "PASS" if overall else "FAIL", "scenarios": results}
with open(os.path.join(out, "results.json"), "w") as f:
    json.dump(doc, f, indent=2)
c = lambda st: sum(r["status"] == st for r in results)
print(f"\nresults.json: {c('PASS')} pass, {c('FAIL')} fail, {c('SKIP')} skip, "
      f"{c('XFAIL')} xfail, {c('XPASS')} xpass → {doc['overall']}")
PY

# ── Unit tests + coverage ────────────────────────────────────────────────────
echo ""
echo "── Unit tests + coverage ──"

# Optional C++ line coverage: rebuild the project's libraries + GTest with gcov
# instrumentation in place (OPTIM/LFLAGS are the MARTe2-sanctioned override
# hooks), let collect.py run the instrumented GTest (emits .gcda) and lcov, then
# restore a clean build so later --skip-build runs aren't left instrumented.
CPP_COV_FLAG=""
if [ "${CPP_COV}" -eq 1 ]; then
    echo "   building instrumented (gcov) libraries + apps + GTest ..."
    COV_O="--coverage"
    COV_L="-Wl,--no-as-needed -fPIC --coverage"
    make -C "${REPO_ROOT}" -f Makefile.gcc clean >/dev/null 2>&1 || true
    # `apps` (StreamHub.ex) must be rebuilt here too, not just `core`: the
    # coverage-pass E2E re-run below launches StreamHub, and `make clean` just
    # removed the non-instrumented StreamHub.ex — without this it would 404 the
    # WS port for every chain/recorder scenario in the coverage pass.
    make -C "${REPO_ROOT}" -f Makefile.gcc core apps TARGET="${TARGET}" \
        OPTIM="${COV_O}" LFLAGS="${COV_L}" 2>&1 | tail -1
    for d in Test/Components/DataSources/UDPStreamer Test/Components/DataSources/UDPStreamerClient Test/Applications/StreamHub Test/GTest Test/Integration; do
        make -C "${REPO_ROOT}/${d}" -f Makefile.gcc TARGET="${TARGET}" \
            OPTIM="${COV_O}" LFLAGS="${COV_L}" 2>&1 | tail -1
    done

    # Coverage-only scenario re-run: the instrumented binaries above are only
    # exercised so far by collect.py's unit-test binaries (MainGTest/Integration),
    # which never touch the E2E-scenario-only code paths (e.g. StreamHub code
    # only hit via the direct/recorder/debug E2E flows). Re-running the full
    # scenario matrix here — against the SAME instrumented binaries, before
    # collect.py's lcov capture — lets those .gcda files accumulate E2E-path
    # coverage too. Run in a subshell with OUT_DIR overridden so the coverage
    # pass's logs/results land in a discardable subdirectory and none of its
    # SCEN_IDS/OUT_DIR/trap state leaks back into this shell (the authoritative
    # results.json was already written above, from the primary pass).
    echo "   re-running scenario matrix under instrumented binaries (coverage pass) ..."
    (
        OUT_DIR="${OUT_DIR}/coverage_pass"
        mkdir -p "${OUT_DIR}"
        run_scenario_matrix "coverage"
    ) || true

    CPP_COV_FLAG="--cpp-coverage"
fi

${PY} "${SCRIPT_DIR}/collect.py" --repo "${REPO_ROOT}" --target "${TARGET}" \
    --out "${OUT_DIR}" --work "${WORK}" ${CPP_COV_FLAG} || true

if [ "${CPP_COV}" -eq 1 ]; then
    echo "   restoring non-instrumented build ..."
    make -C "${REPO_ROOT}" -f Makefile.gcc clean >/dev/null 2>&1 || true
    make -C "${REPO_ROOT}" -f Makefile.gcc core apps TARGET="${TARGET}" 2>&1 | tail -1
    (cd "${SCRIPT_DIR}/client" && go build -o chain-client . >/dev/null 2>&1) || true
    (cd "${SCRIPT_DIR}/debugclient" && go build -o debugclient . >/dev/null 2>&1) || true
fi

# ── Consolidated report data (+ history/regression + trend plots) ────────────
${PY} "${SCRIPT_DIR}/report_build.py" --repo "${REPO_ROOT}" \
    --results "${OUT_DIR}/results.json" --work "${WORK}" --out "${OUT_DIR}" \
    --stress-results "${OUT_DIR}/stress/stress_results.json" || true

# ── PDF ──────────────────────────────────────────────────────────────────────
if command -v typst >/dev/null 2>&1 && [ -f "${SCRIPT_DIR}/E2E_Report.typ" ]; then
    cp "${SCRIPT_DIR}/E2E_Report.typ" "${OUT_DIR}/" 2>/dev/null || true
    # Embedded waveform overviews live in WORK; the template references them by
    # bare name, so stage them next to report_data.json before compiling.
    cp "${WORK}"/wave_*.png "${OUT_DIR}/" 2>/dev/null || true
    if (cd "${OUT_DIR}" && typst compile E2E_Report.typ E2E_Report.pdf); then
        echo "PDF: ${OUT_DIR}/E2E_Report.pdf"
    else
        echo "typst compile failed (report_data.json present at ${OUT_DIR})"
    fi
fi

echo "Done — artifacts in ${OUT_DIR} and ${WORK}"