Implemented better testing and fixed skipepd frames

Test/E2E/chain/collect.py

@@ -21,6 +21,7 @@ import os
 import re
 import subprocess
 import sys
+import time
 import xml.etree.ElementTree as ET
 
 
@@ -60,18 +61,88 @@ def gtest_suite(gtest_bin, work):
     return s
 
 
-# ── Go ────────────────────────────────────────────────────────────────────────
+# ── C++ Integration (DebugService runtime, non-GTest) ─────────────────────────
 
-def go_suite(client_dir, work):
-    s = {"name": "Go (chain-client)", "lang": "go", "total": 0, "passed": 0,
+def integration_suite(int_bin, work, timeout=220):
+    """Run the printf-narrated IntegrationTests.ex binary and heuristically
+    derive per-test pass/fail from its stdout.
+
+    This binary predates GTest adoption and always ``return 0`` from main()
+    (only an internal 180s SIGALRM timeout or an OS-level crash produce a
+    non-zero exit), so exit code alone is not a reliable signal. Each of its
+    7 "--- Test N: ..." blocks prints "SUCCESS:"/"VALIDATION SUCCESSFUL:" on
+    success or "ERROR:"/"FAILURE:" on failure, so split stdout by those
+    headers and flag a block failed if it contains an ERROR/FAILURE marker.
+    Exercises DebugServiceBase.cpp/DebugService.cpp runtime logic that the
+    header-only DebugServiceGTest suite deliberately does not touch, so it is
+    the only source of real coverage for those files.
+    """
+    s = {"name": "C++ Integration", "lang": "cpp", "total": 0, "passed": 0,
          "failed": 0, "skipped": 0, "time_s": 0.0, "ok": False, "avail": False}
-    cov_p = os.path.join(work, "go_cover.out")
-    rc, out, err = _run(["go", "test", "-json", f"-coverprofile={cov_p}", "./..."],
-                        cwd=client_dir)
-    if rc == 127:
-        s["detail"] = "go toolchain not found"
+    if not int_bin or not os.path.exists(int_bin):
+        s["detail"] = "IntegrationTests binary not found"
         return s
     s["avail"] = True
+    t0 = time.time()
+    rc, out, err = _run([int_bin], timeout=timeout)
+    s["time_s"] = round(time.time() - t0, 1)
+    text = out + "\n" + err
+    blocks = re.split(r"\n(?=--- Test \d+:)", text)
+    test_blocks = [b for b in blocks if b.lstrip().startswith("--- Test")]
+    finished = "All Integration Tests Finished." in text
+    s["total"] = len(test_blocks)
+    s["failed"] = sum(1 for b in test_blocks if re.search(r"\b(ERROR|FAILURE):", b))
+    if not finished and s["total"] == 0:
+        # Crashed/timed out before printing anything useful.
+        s["total"] = 1
+        s["failed"] = 1
+        s["detail"] = f"binary did not complete (rc={rc}): {(err or out)[-200:]}"
+    elif not finished:
+        s["detail"] = f"binary exited rc={rc} before finishing all tests"
+    s["passed"] = s["total"] - s["failed"]
+    s["ok"] = finished and s["failed"] == 0 and s["total"] > 0
+    return s
+
+
+# ── Go ────────────────────────────────────────────────────────────────────────
+
+def go_all_suites(repo, work):
+    """Run Go test suites across all project modules and aggregate results."""
+    modules = [
+        (os.path.join(repo, "Test/E2E/chain/client"),
+         "Go (chain-client)"),
+        (os.path.join(repo, "Common/Client/go"),
+         "Go (common udpsprotocol + wshub)"),
+        (os.path.join(repo, "Client/debugger"),
+         "Go (debugger)"),
+    ]
+    total_pct = 0.0
+    pct_count = 0
+    suites = []
+    for mod_dir, name in modules:
+        s = {"name": name, "lang": "go", "total": 0, "passed": 0,
+             "failed": 0, "skipped": 0, "time_s": 0.0, "ok": False, "avail": False}
+        cov_p = os.path.join(work, f"go_cover_{name.replace(' ', '_')}.out")
+        rc, out, err = _run(
+            ["go", "test", "-json", f"-coverprofile={cov_p}", "./..."],
+            cwd=mod_dir)
+        if rc == 127:
+            s["detail"] = "go toolchain not found"
+            suites.append(s)
+            continue
+        s["avail"] = True
+        cov_pct = _parse_go_json(out, s)
+        if cov_pct is not None:
+            s["cov_pct"] = cov_pct
+            total_pct += cov_pct
+            pct_count += 1
+        suites.append(s)
+    return suites, (round(total_pct / pct_count, 1) if pct_count else None)
+
+
+def _parse_go_json(out, s):
+    """Parse Go test -json output into passed/failed/skipped counts.
+    Returns coverage percentage (float or None)."""
     cov_pct = None
     for line in out.splitlines():
         try:
@@ -93,8 +164,7 @@ def go_suite(client_dir, work):
             if m:
                 cov_pct = float(m.group(1))
     s["ok"] = s["failed"] == 0 and s["total"] > 0
-    s["cov_pct"] = cov_pct
-    return s
+    return cov_pct
 
 
 # ── Python ──────────────────────────────────────────────────────────────────
@@ -250,12 +320,16 @@ def main():
     work = args.work or args.out
     os.makedirs(work, exist_ok=True)
     chain_dir = os.path.dirname(os.path.abspath(__file__))
-    client_dir = os.path.join(chain_dir, "client")
     gtest_bin = os.path.join(args.repo, "Build", args.target, "GTest", "MainGTest.ex")
+    # BUILD_DIR for Test/Integration doubles the last path component
+    # ($(PACKAGE)/$(lastword of CURDIR)) — see MakeStdLibDefs.gcc.
+    integration_bin = os.path.join(args.repo, "Build", args.target,
+                                    "Test", "Integration", "Integration",
+                                    "IntegrationTests.ex")
 
-    suites = [gtest_suite(gtest_bin, work),
-              go_suite(client_dir, work),
-              py_suite(chain_dir, work)]
+    go_suites, go_avg_cov = go_all_suites(args.repo, work)
+    suites = ([gtest_suite(gtest_bin, work), integration_suite(integration_bin, work)]
+              + go_suites + [py_suite(chain_dir, work)])
     totals = {k: sum(s.get(k, 0) for s in suites)
               for k in ("total", "passed", "failed", "skipped")}
     ut = {"suites": suites, "totals": totals,
@@ -265,11 +339,10 @@ def main():
 
     langs = []
     py = next(s for s in suites if s["lang"] == "python")
-    go = next(s for s in suites if s["lang"] == "go")
     langs.append({"name": "Python", "avail": py.get("cov_pct") is not None,
                   "pct": py.get("cov_pct"), "note": "coverage.py"})
-    langs.append({"name": "Go", "avail": go.get("cov_pct") is not None,
-                  "pct": go.get("cov_pct"), "note": "go test -cover"})
+    langs.append({"name": "Go", "avail": go_avg_cov is not None,
+                  "pct": go_avg_cov, "note": "go test -cover (avg across modules)"})
     cpp = cpp_coverage(args.repo, args.target) if args.cpp_coverage else \
         {"name": "C++", "avail": False, "pct": None, "note": "skipped (use --cpp-coverage)"}
     langs.append(cpp)

Test/E2E/chain/run_chain_e2e.sh

@@ -24,7 +24,7 @@ mkdir -p "${OUT_DIR}" "${WORK}"
 SKIP_BUILD=0
 ONLY=""
 PDF_ONLY=0
-CPP_COV=0
+CPP_COV=1
 while [ $# -gt 0 ]; do
     case "$1" in
         --skip-build) SKIP_BUILD=1 ;;
@@ -254,7 +254,7 @@ if [ "${CPP_COV}" -eq 1 ]; then
     make -C "${REPO_ROOT}" -f Makefile.gcc clean >/dev/null 2>&1 || true
     make -C "${REPO_ROOT}" -f Makefile.gcc core TARGET="${TARGET}" \
         OPTIM="${COV_O}" LFLAGS="${COV_L}" 2>&1 | tail -1
-    for d in Test/Components/DataSources/UDPStreamer Test/Applications/StreamHub Test/GTest; do
+    for d in Test/Components/DataSources/UDPStreamer Test/Components/DataSources/UDPStreamerClient Test/Applications/StreamHub Test/GTest Test/Integration; do
         make -C "${REPO_ROOT}/${d}" -f Makefile.gcc TARGET="${TARGET}" \
             OPTIM="${COV_O}" LFLAGS="${COV_L}" 2>&1 | tail -1
     done

Test/E2E/chain/validate_waveform.py

@@ -21,6 +21,14 @@ Oracle (per signal)
   phase offset automatically). nRMSE tolerance is relaxed by the quant step.
 * **Fed reference** (when ``--tap`` given): each received value must be within
   ``tol`` of some tap value too.
+* **Continuity** (always, ≥10 points): a stream that stalls (client falling
+  behind, hub failing to flush a window, ...) can still pass fidelity —
+  whatever few samples *did* arrive still match the ground truth — while the
+  plot shows gaping holes. Flags any inter-sample gaps that are >10x the
+  median spacing and fails when their *summed* duration exceeds 5% of the
+  capture span. Calibrated against the full scenario matrix: healthy streams
+  (bursty per-tick live pushes, decimation, fragmentation, multicast, ...) top
+  out at 0.7% outlier-gap time; a stalled stream showed 55-91%.
 """
 import argparse
 import json
@@ -122,6 +130,29 @@ def nearest_err(recv_v, truth_v):
     return float(np.max(d)) if d.size else 0.0
 
 
+def gap_check(t_recv, outlier_mult=10.0, max_outlier_frac=0.05):
+    """Detect large discontiguous holes in a received time series.
+
+    A handful of gaps a few times the median spacing are normal (bursty
+    per-tick live pushes, decimation, LTTB). Returns (ok, gap_frac, n_gaps,
+    max_gap): ``gap_frac`` is the fraction of the total capture span consumed
+    by gaps larger than ``outlier_mult`` times the median inter-sample gap;
+    when that adds up to more than ``max_outlier_frac`` of the whole capture,
+    the stream stalled/dropped a chunk rather than merely being decimated.
+    """
+    if t_recv.size < 10:
+        return True, 0.0, 0, 0.0
+    t = np.sort(t_recv.astype(np.float64))
+    dt = np.diff(t)
+    span = float(t[-1] - t[0])
+    med = float(np.median(dt))
+    if span <= 0.0 or med <= 0.0:
+        return True, 0.0, 0, 0.0
+    outliers = dt[dt > outlier_mult * med]
+    gap_frac = float(outliers.sum() / span)
+    return gap_frac <= max_outlier_frac, gap_frac, int(outliers.size), float(dt.max())
+
+
 def sine_shape(t, v, freq):
     """Return (corr, nrmse, amp_fit) for a sinusoid fit at ``freq``."""
     w = 2.0 * np.pi * freq
@@ -153,6 +184,13 @@ def compare_signal(gt, t_recv, v_recv, tap_v=None):
     fidelity_ok = max_err <= tol
     m["fidelity_ok"] = bool(fidelity_ok)
 
+    gap_ok, gap_frac, n_gaps, max_gap = gap_check(t_recv)
+    m.update(gap_ok=bool(gap_ok), gap_frac=round(gap_frac, 4),
+              n_gaps=n_gaps, max_gap=max_gap)
+    if not gap_ok:
+        m["reason"] = (f"data hole: {gap_frac:.1%} of capture span in "
+                        f"{n_gaps} gaps >10x median spacing (max={max_gap:.4g}s)")
+
     shape_ok = True
     if gt["formula"] == "sine" and v_recv.size >= 8 and gt["freq"]:
         corr, nrmse, amp = sine_shape(t_recv, v_recv, gt["freq"])
@@ -180,7 +218,7 @@ def compare_signal(gt, t_recv, v_recv, tap_v=None):
         fed_ok = fed_err <= tol
         m.update(fed_err=fed_err, fed_ok=bool(fed_ok))
 
-    m["pass"] = bool(fidelity_ok and shape_ok and fed_ok)
+    m["pass"] = bool(fidelity_ok and shape_ok and fed_ok and gap_ok)
     return m