feat(tcp): TCPStreamer DataSources, in-process TCPStreamService, stream multiplexing and TCPStreamJoin

Add TCPStreamer/TCPStreamerClient (TCP transport behind the UDPSServerI/UDPSClientI
seam), an optional single in-process TCPStreamService serving many streams over
worker threads, multiplexed SUBSCRIBE (comma list, stream index in fragmentIdx),
and TCPStreamJoin, which merges several streams into one sample-and-hold stream
paced by a master. StreamHub takes a comma list of streams over one connection.
Includes GTests, E2E scenarios s58-s73 and docs.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Test/E2E/suite/gen_cfg.py

@@ -103,10 +103,23 @@ def _streamer_sig(sig):
     return f"{sig['name']} = {{ {' '.join(parts)} }}"
 
 
+SERVICE_NAME = "Svc"
+
+
+def _service_port(scenario):
+    """Port of the shared TCPStreamService (first source's port)."""
+    return scenario["sources"][0]["udp_port"]
+
+
 def _streamer_block(src, scenario):
     s = src["signals"]
-    parts = [f"Port = {src['udp_port']}",
-             f"MaxPayloadSize = {scenario['max_payload']}",
+    net = scenario["network"]
+    if net == "tcp_service":
+        # Port, MaxClients, ... belong to the service; the streamer only joins it.
+        parts = [f'Service = "{SERVICE_NAME}"', f'StreamName = "{S.stream_name(scenario, src)}"']
+    else:
+        parts = [f"Port = {src['udp_port']}"]
+    parts += [f"MaxPayloadSize = {scenario['max_payload']}",
              f'PublishingMode = "{scenario["publishing"]}"']
     if scenario["publishing"] == "Accumulate":
         parts.append(f"MinRefreshRate = {scenario['min_refresh_hz']}")
@@ -118,7 +131,8 @@ def _streamer_block(src, scenario):
         iface = src.get("interface") or _mcast_interface_ip(src["multicast_group"])
         parts.append(f'Interface = "{iface}"')
     sigs = " ".join(_streamer_sig(sig) for sig in s)
-    return (f"        +Streamer_{src['id']} = {{ Class = UDPStreamer "
+    cls = "UDPStreamer" if net in ("unicast", "multicast") else "TCPStreamer"
+    return (f"        +Streamer_{src['id']} = {{ Class = {cls} "
             f"{' '.join(parts)} Signals = {{ {sigs} }} }}")
 
 
@@ -202,7 +216,19 @@ def write_marte_cfg(scenario, path, input_bin, tap_bin=None):
     data_block = "\n".join(datas)
     thread_list = " ".join(thread_funcs)
 
-    cfg = f"""$ChainE2E = {{
+    service_block = ""
+    if scenario["network"] == "tcp_service":
+        service_block = (f"+{SERVICE_NAME} = {{ Class = TCPStreamService "
+                         f"Port = {_service_port(scenario)} Workers = 2 "
+                         f"MaxClients = 16 MaxStreams = 16 }}\n")
+        hub = scenario.get("hub")
+        if hub and hub["mode"] == "join":
+            names = ",".join(S.stream_name(scenario, x) for x in scenario["sources"])
+            service_block += (f"+Join = {{ Class = TCPStreamJoin Service = \"{SERVICE_NAME}\" "
+                              f"StreamName = \"{hub['name']}\" Sources = \"{names}\" "
+                              f"Master = \"{hub['master']}\" }}\n")
+
+    cfg = f"""{service_block}$ChainE2E = {{
     Class = RealTimeApplication
     +Functions = {{
         Class = ReferenceContainer
@@ -227,10 +253,26 @@ def write_marte_cfg(scenario, path, input_bin, tap_bin=None):
 def write_hub_cfg(scenario, path):
     os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
     src_blocks = []
-    for src in scenario["sources"]:
+    hub = scenario.get("hub")
+    if hub and hub["mode"] == "mux":
+        names = ",".join(S.stream_name(scenario, x) for x in scenario["sources"])
+        src_blocks.append(
+            f'        {hub["node"]} = {{ Label = "chain" Addr = "127.0.0.1" '
+            f'Port = {_service_port(scenario)} Transport = "tcp" StreamName = "{names}" }}')
+    elif hub and hub["mode"] == "join":
+        src_blocks.append(
+            f'        {hub["name"]} = {{ Label = "chain {hub["name"]}" Addr = "127.0.0.1" '
+            f'Port = {_service_port(scenario)} Transport = "tcp" StreamName = "{hub["name"]}" }}')
+    for src in ([] if hub else scenario["sources"]):
+        net = scenario["network"]
+        port = _service_port(scenario) if net == "tcp_service" else src["udp_port"]
         parts = [f'Label = "chain {src["id"]}"', 'Addr = "127.0.0.1"',
-                 f"Port = {src['udp_port']}"]
-        if scenario["network"] == "multicast":
+                 f"Port = {port}"]
+        if net in ("tcp", "tcp_service"):
+            parts.append('Transport = "tcp"')
+        if net == "tcp_service":
+            parts.append(f'StreamName = "{src["id"]}"')
+        if net == "multicast":
             parts.append(f'MulticastGroup = "{src["multicast_group"]}"')
             parts.append(f"DataPort = {src['data_port']}")
         src_blocks.append(f"        {src['id']} = {{ {' '.join(parts)} }}")

Test/E2E/suite/gen_data.py

@@ -113,6 +113,12 @@ def build_ground_truth(scenario):
                 "range_min": sig["range_min"], "range_max": sig["range_max"],
                 "is_time": sig["is_time"],
             }
+    hub = scenario.get("hub")
+    if hub and hub.get("mode") == "join":
+        # The merged stream carries every source signal as "<src>_<signal>".
+        for key in list(gt):
+            src_id, sig_name = key.split(":", 1)
+            gt[f"{hub['name']}:{src_id}_{sig_name}"] = gt[key]
     return gt
 
 

Test/E2E/suite/run_e2e.sh

@@ -59,6 +59,9 @@ COMP="${MARTe2_Components_DIR}/Build/${TARGET}/Components"
 export LD_LIBRARY_PATH="\
 ${BUILD_DIR}/Components/DataSources/UDPStreamer:\
 ${BUILD_DIR}/Components/DataSources/UDPStreamerClient:\
+${BUILD_DIR}/Components/DataSources/TCPStreamer:\
+${BUILD_DIR}/Components/DataSources/TCPStreamerClient:\
+${BUILD_DIR}/Components/Interfaces/TCPStreamService:\
 ${BUILD_DIR}/Components/Interfaces/UDPStream:\
 ${BUILD_DIR}/Components/Interfaces/DebugService:\
 ${BUILD_DIR}/Components/Interfaces/TCPLogger:\
@@ -68,6 +71,15 @@ ${COMP}/DataSources/FileDataSource:\
 ${COMP}/GAMs/IOGAM:\
 ${LD_LIBRARY_PATH:-}"
 
+# MARTe2 resolves a class by dlopen("<Class>.so"), but the FileDataSource library
+# bundles FileReader and FileWriter under one name. Provide per-class aliases.
+ALIAS_DIR="${WORK}/class_aliases"
+mkdir -p "${ALIAS_DIR}"
+for cls in FileReader FileWriter; do
+    ln -sf "${COMP}/DataSources/FileDataSource/FileDataSource.so" "${ALIAS_DIR}/${cls}.so"
+done
+export LD_LIBRARY_PATH="${ALIAS_DIR}:${LD_LIBRARY_PATH}"
+
 MARTE_APP="${MARTe2_DIR}/Build/${TARGET}/App/MARTeApp.ex"
 STREAMHUB_EX="${BUILD_DIR}/StreamHub/StreamHub.ex"
 CLIENT="${SCRIPT_DIR}/client/chain-client"
@@ -94,6 +106,9 @@ if [ "${SKIP_BUILD}" -eq 0 ]; then
     make -C "${REPO_ROOT}/Source/Components/Interfaces/UDPStream" -f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -1
     make -C "${REPO_ROOT}/Source/Components/DataSources/UDPStreamer" -f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -1
     make -C "${REPO_ROOT}/Source/Components/DataSources/UDPStreamerClient" -f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -1
+    make -C "${REPO_ROOT}/Source/Components/Interfaces/TCPStreamService" -f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -1
+    make -C "${REPO_ROOT}/Source/Components/DataSources/TCPStreamer" -f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -1
+    make -C "${REPO_ROOT}/Source/Components/DataSources/TCPStreamerClient" -f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -1
     make -C "${REPO_ROOT}/Source/Applications/StreamHub" -f Makefile.gcc TARGET="${TARGET}" 2>&1 | tail -1
 fi
 if [ ! -x "${CLIENT}" ]; then

Test/E2E/suite/scenarios.py

@@ -15,7 +15,21 @@ Scenario dict schema
 {
   "id":              str   # unique slug, also the artifact basename
   "desc":            str   # human-readable one-liner for the report
-  "network":         "unicast" | "multicast"
+  "network":         "unicast" | "multicast" | "tcp" | "tcp_service"
+                     # tcp:         one TCPStreamer per source, each listening on
+                     #              its own udp_port
+                     # tcp_service: one in-process TCPStreamService (root object)
+                     #              on sources[0].udp_port; every source is a
+                     #              TCPStreamer joined to it with StreamName=<src id>
+  "hub":             None | {"mode": "mux", "node": str}
+                     | {"mode": "join", "name": str, "master": str}
+                     # tcp_service only. mux:  the hub takes every source over ONE
+                     #   multiplexed connection (a single comma separated StreamName);
+                     #   source ids must be "<node>_<stream>" (the hub names its sessions
+                     #   "<node>_<stream>") and the MARTe stream name is "<stream>".
+                     # join: a TCPStreamJoin merges all sources into stream <name>
+                     #   (paced by <master>); the hub subscribes to <name> only and sees
+                     #   signals "<src>_<signal>" under hub source <name>.
   "publishing":      "Strict" | "Accumulate" | "Decimate"
   "ratio":           int | None        # Decimate only (>=1)
   "min_refresh_hz":  float | None       # Accumulate only (>0)
@@ -132,6 +146,14 @@ def _sig(name, type, elements=1, time_mode="PacketTime", time_signal=None,
     }
 
 
+def stream_name(scenario, src):
+    """MARTe-side TCPStreamService stream name of a source."""
+    hub = scenario.get("hub")
+    if hub and hub.get("mode") == "mux":
+        return src["id"][len(hub["node"]) + 1:]
+    return src["id"]
+
+
 def validate_scenario(s):
     """Return a list of validity error strings (empty == valid)."""
     errs = []
@@ -152,8 +174,8 @@ def validate_scenario(s):
         errs.append(f"unknown scenario kind: {kind}")
         return errs
     row_dt, num_rows, _producer_hz, loop_hz = geometry(s)
-    if s.get("network") not in ("unicast", "multicast"):
-        errs.append("network must be unicast|multicast")
+    if s.get("network") not in ("unicast", "multicast", "tcp", "tcp_service"):
+        errs.append("network must be unicast|multicast|tcp|tcp_service")
     if s.get("publishing") not in ("Strict", "Accumulate", "Decimate"):
         errs.append("publishing invalid")
     if s["publishing"] == "Decimate" and not (s.get("ratio") and s["ratio"] >= 1):
@@ -164,11 +186,27 @@ def validate_scenario(s):
     if not s.get("sources"):
         errs.append("no sources")
     max_payload = s.get("max_payload")
-    if max_payload is not None and max_payload + UDPS_HEADER_SIZE > MAX_UDP_PAYLOAD:
+    is_udp = s["network"] in ("unicast", "multicast")
+    if is_udp and max_payload is not None and max_payload + UDPS_HEADER_SIZE > MAX_UDP_PAYLOAD:
         errs.append(
             f"max_payload {max_payload} + {UDPS_HEADER_SIZE}B header exceeds the "
             f"{MAX_UDP_PAYLOAD}B UDP datagram limit (sendto EMSGSIZE); "
             f"cap at {MAX_UDP_PAYLOAD - UDPS_HEADER_SIZE}")
+    hub = s.get("hub")
+    if hub:
+        ids = [x["id"] for x in s.get("sources", [])]
+        if s["network"] != "tcp_service":
+            errs.append("hub mux/join needs network=tcp_service")
+        elif hub.get("mode") == "mux":
+            if not all(i.startswith(hub["node"] + "_") for i in ids):
+                errs.append(f"mux source ids must start with '{hub['node']}_'")
+        elif hub.get("mode") == "join":
+            if hub.get("master") not in ids or hub.get("name") in ids:
+                errs.append("join master must be a source id and the name must differ from all of them")
+            if s["publishing"] == "Accumulate":
+                errs.append("join cannot merge Accumulate sources")
+        else:
+            errs.append("hub.mode must be mux|join")
     for src in s.get("sources", []):
         if s["network"] == "multicast":
             if not src.get("multicast_group") or not src.get("data_port"):
@@ -250,7 +288,7 @@ def _src(sid, signals, multicast=False):
 def mk(sid, desc, sources, network="unicast", publishing="Strict",
        ratio=None, min_refresh_hz=None, max_payload=1400,
        oracle="analytic", checks=("live", "zoom"), trig=None,
-       known_issue=None, row_dt=None, num_rows=None, producer_hz=None):
+       known_issue=None, row_dt=None, num_rows=None, producer_hz=None, hub=None):
     # row_dt / num_rows / producer_hz override the global producer geometry for
     # this scenario only (default None == use the suite-wide NUM_ROWS / ROW_DT /
     # 1 kHz). They are kept together so the per-cycle wall gap (1/producer_hz),
@@ -264,6 +302,7 @@ def mk(sid, desc, sources, network="unicast", publishing="Strict",
         "client_checks": list(checks), "trig_signal": trig,
         "known_issue": known_issue,
         "row_dt": row_dt, "num_rows": num_rows, "producer_hz": producer_hz,
+        "hub": hub,
     }
 
 
@@ -577,9 +616,94 @@ _HIGHRATE = [
        checks=("live", "zoom")),
 ]
 
+# ── TCP transports ────────────────────────────────────────────────────────────
+# Same chain, TCPStreamer instead of UDPStreamer. "tcp": one listening socket per
+# source. "tcp_service": one TCPStreamService (in-process, root object) serves all
+# sources; each is a TCPStreamer joined with Service/StreamName and the hub
+# subscribes to it by name. TCP does not fragment (max_payload only sizes
+# Accumulate batches), so the UDP datagram cap does not apply.
+_TCP = [
+    mk("s60_tcp_scalar", "TCPStreamer: float32 sine 5 Hz + uint32 counter, all checks",
+       [_src("src", [_sig("Counter", "uint32", 1, formula="counter"),
+                     _sig("Sine", "float32", 1, formula="sine", freq=5.0, unit="V")])],
+       network="tcp", checks=("live", "zoom", "window", "trigger"), trig="src:Sine"),
+    mk("s61_tcp_arr_fullarray", "TCPStreamer: 32-elem float32 FullArray sine 5 Hz",
+       [_src("src", [
+           _sig("TimeArr", "uint64", 32, unit="ns", formula="time_ns", is_time=True),
+           _sig("Wave", "float32", 32, time_mode="FullArray",
+                time_signal="TimeArr", formula="sine", freq=5.0),
+       ])],
+       network="tcp", oracle="both"),
+    mk("s62_tcp_accumulate", "TCPStreamer: Accumulate @100 Hz scalar sine 5 Hz",
+       [_src("src", [_sig("Sine", "float32", 1, formula="sine", freq=5.0)])],
+       network="tcp", publishing="Accumulate", min_refresh_hz=100.0),
+    mk("s63_tcp_decimate_quant", "TCPStreamer: Decimate 2 + 16-elem quant uint16 sine 5 Hz",
+       [_src("src", [_sig("Wave", "float32", 16, quant="uint16", range_min=-1.0,
+                          range_max=1.0, formula="sine", freq=5.0)])],
+       network="tcp", publishing="Decimate", ratio=2),
+    mk("s64_tcp_multisrc", "TCPStreamer: two sources on separate ports",
+       [_src("a", [_sig("Sine", "float32", 1, formula="sine", freq=5.0)]),
+        _src("b", [_sig("Cnt", "uint32", 1, formula="counter")])],
+       network="tcp"),
+    mk("s65_tcp_highrate",
+       "TCPStreamer: 4x float32 2000-elem arrays, FirstSample 200 kSps, 100 Hz "
+       "(~3 MB/s, 32 KB frames)",
+       [_src("src", [
+           _sig("Tns", "uint64", 1, unit="ns", formula="time_ns", is_time=True),
+       ] + [
+           _sig(f"S{i}", "float32", 2000, time_mode="FirstSample",
+                time_signal="Tns", sampling_rate=2.0e5, formula="ramp")
+           for i in range(4)
+       ])],
+       network="tcp", max_payload=40000, row_dt=0.01, num_rows=50, producer_hz=100,
+       checks=("live", "zoom")),
+    mk("s66_svc_scalar", "TCPStreamService: one stream, sine 5 Hz + counter, all checks",
+       [_src("src", [_sig("Counter", "uint32", 1, formula="counter"),
+                     _sig("Sine", "float32", 1, formula="sine", freq=5.0, unit="V")])],
+       network="tcp_service", checks=("live", "zoom", "window", "trigger"),
+       trig="src:Sine"),
+    mk("s67_svc_multisrc",
+       "TCPStreamService: three streams of one service (int16 ramp / float64 sine / uint8 counter)",
+       [_src("a", [_sig("Ramp", "int16", 1, formula="ramp")]),
+        _src("b", [_sig("Sine", "float64", 1, formula="sine", freq=5.0)]),
+        _src("c", [_sig("Cnt", "uint8", 1, formula="counter")])],
+       network="tcp_service"),
+    mk("s68_svc_arr_fullarray", "TCPStreamService: 32-elem float32 FullArray sine 5 Hz",
+       [_src("src", [
+           _sig("TimeArr", "uint64", 32, unit="ns", formula="time_ns", is_time=True),
+           _sig("Wave", "float32", 32, time_mode="FullArray",
+                time_signal="TimeArr", formula="sine", freq=5.0),
+       ])],
+       network="tcp_service", oracle="both"),
+    mk("s69_svc_accumulate", "TCPStreamService: Accumulate @100 Hz, two streams",
+       [_src("a", [_sig("Sine", "float32", 1, formula="sine", freq=5.0)]),
+        _src("b", [_sig("Sine", "float32", 1, formula="sine", freq=10.0)])],
+       network="tcp_service", publishing="Accumulate", min_refresh_hz=100.0),
+    mk("s70_svc_decimate_quant", "TCPStreamService: Decimate 2 + 16-elem quant uint16 sine 5 Hz",
+       [_src("src", [_sig("Wave", "float32", 16, quant="uint16", range_min=-1.0,
+                          range_max=1.0, formula="sine", freq=5.0)])],
+       network="tcp_service", publishing="Decimate", ratio=2),
+    mk("s71_svc_six_streams",
+       "TCPStreamService: six concurrent streams share two worker threads",
+       [_src(f"s{i}", [_sig("Sine", "float32", 1, formula="sine", freq=5.0 * (i + 1))])
+        for i in range(6)],
+       network="tcp_service"),
+    mk("s72_svc_mux",
+       "TCPStreamService: hub takes three streams over ONE multiplexed connection",
+       [_src("mux_a", [_sig("Ramp", "int16", 1, formula="ramp")]),
+        _src("mux_b", [_sig("Sine", "float64", 1, formula="sine", freq=5.0)]),
+        _src("mux_c", [_sig("Cnt", "uint8", 1, formula="counter")])],
+       network="tcp_service", hub={"mode": "mux", "node": "mux"}),
+    mk("s73_svc_join",
+       "TCPStreamJoin: two streams merged into one (master b), hub subscribes to the merge",
+       [_src("a", [_sig("Counter", "uint32", 1, formula="counter")]),
+        _src("b", [_sig("Sine", "float32", 1, formula="sine", freq=5.0, unit="V")])],
+       network="tcp_service", hub={"mode": "join", "name": "all", "master": "b"}),
+]
+
 SCENARIOS = (_STARTERS + _TYPES + _ARRAYS + _TIMEMODES + _QUANT + _PUBLISH +
              _PAYLOAD + _MCAST + _MULTISRC + _INTERACT + _TRIGGER + _MISC +
-             _HIGHRATE)
+             _HIGHRATE + _TCP)
 
 for _s in SCENARIOS:
     _s.setdefault("kind", "chain")
@@ -606,6 +730,23 @@ _DIRECT = [
         "client_checks": [],
         "known_issue": None,
     },
+    {
+        "id": "s58_direct_tcp",
+        "desc": "Direct TCPStreamer->TCPStreamerClient round-trip",
+        "kind": "direct",
+        "cfg": "Test/E2E/datasources/E2ETCPTest.cfg",
+        "client_checks": [],
+        "known_issue": None,
+    },
+    {
+        "id": "s59_direct_tcp_service",
+        "desc": "Direct TCPStreamer (in-process TCPStreamService) -> TCPStreamerClient "
+                "(named-stream subscription) round-trip",
+        "kind": "direct",
+        "cfg": "Test/E2E/datasources/E2ETCPServiceTest.cfg",
+        "client_checks": [],
+        "known_issue": None,
+    },
 ]
 
 _RECORDER = [

Test/E2E/suite/tests_py.py

@@ -68,7 +68,8 @@ class TestScenarios(unittest.TestCase):
         self.assertIn("direct", kinds)
         self.assertIn("recorder", kinds)
         direct_ids = {s["id"] for s in S.SCENARIOS if s["kind"] == "direct"}
-        self.assertEqual(direct_ids, {"s52_direct_unicast", "s53_direct_multicast"})
+        self.assertEqual(direct_ids, {"s52_direct_unicast", "s53_direct_multicast",
+                                    "s58_direct_tcp", "s59_direct_tcp_service"})
         recorder_ids = {s["id"] for s in S.SCENARIOS if s["kind"] == "recorder"}
         self.assertEqual(recorder_ids, {"s54_recorder"})