martino/MARTe-Integrated-Components
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(tcp): TCPStreamer DataSources, in-process TCPStreamService, stream multiplexing and TCPStreamJoin

Browse Source 
Add TCPStreamer/TCPStreamerClient (TCP transport behind the UDPSServerI/UDPSClientI
seam), an optional single in-process TCPStreamService serving many streams over
worker threads, multiplexed SUBSCRIBE (comma list, stream index in fragmentIdx),
and TCPStreamJoin, which merges several streams into one sample-and-hold stream
paced by a master. StreamHub takes a comma list of streams over one connection.
Includes GTests, E2E scenarios s58-s73 and docs.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Martino FerrariandClaude Opus 4.6 committed 2026-11-06 13:41:27 +01:00
1 parent 5562877c99
commit 50d85363a3
79 files changed
+12348 -138
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Test/E2E/suite/scenarios.py
+147 -6
View File
@@ -15,7 +15,21 @@ Scenario dict schema
{
"id": str # unique slug, also the artifact basename
"desc": str # human-readable one-liner for the report
"network": "unicast" | "multicast"
"network": "unicast" | "multicast" | "tcp" | "tcp_service"
# tcp: one TCPStreamer per source, each listening on
# its own udp_port
# tcp_service: one in-process TCPStreamService (root object)
# on sources[0].udp_port; every source is a
# TCPStreamer joined to it with StreamName=<src id>
"hub": None | {"mode": "mux", "node": str}
| {"mode": "join", "name": str, "master": str}
# tcp_service only. mux: the hub takes every source over ONE
# multiplexed connection (a single comma separated StreamName);
# source ids must be "<node>_<stream>" (the hub names its sessions
# "<node>_<stream>") and the MARTe stream name is "<stream>".
# join: a TCPStreamJoin merges all sources into stream <name>
# (paced by <master>); the hub subscribes to <name> only and sees
# signals "<src>_<signal>" under hub source <name>.
"publishing": "Strict" | "Accumulate" | "Decimate"
"ratio": int | None # Decimate only (>=1)
"min_refresh_hz": float | None # Accumulate only (>0)
@@ -132,6 +146,14 @@ def _sig(name, type, elements=1, time_mode="PacketTime", time_signal=None,
}
def stream_name(scenario, src):
"""MARTe-side TCPStreamService stream name of a source."""
hub = scenario.get("hub")
if hub and hub.get("mode") == "mux":
return src["id"][len(hub["node"]) + 1:]
return src["id"]
def validate_scenario(s):
"""Return a list of validity error strings (empty == valid)."""
errs = []
@@ -152,8 +174,8 @@ def validate_scenario(s):
errs.append(f"unknown scenario kind: {kind}")
return errs
row_dt, num_rows, _producer_hz, loop_hz = geometry(s)
if s.get("network") not in ("unicast", "multicast"):
errs.append("network must be unicast|multicast")
if s.get("network") not in ("unicast", "multicast", "tcp", "tcp_service"):
errs.append("network must be unicast|multicast|tcp|tcp_service")
if s.get("publishing") not in ("Strict", "Accumulate", "Decimate"):
errs.append("publishing invalid")
if s["publishing"] == "Decimate" and not (s.get("ratio") and s["ratio"] >= 1):
@@ -164,11 +186,27 @@ def validate_scenario(s):
if not s.get("sources"):
errs.append("no sources")
max_payload = s.get("max_payload")
if max_payload is not None and max_payload + UDPS_HEADER_SIZE > MAX_UDP_PAYLOAD:
is_udp = s["network"] in ("unicast", "multicast")
if is_udp and max_payload is not None and max_payload + UDPS_HEADER_SIZE > MAX_UDP_PAYLOAD:
errs.append(
f"max_payload {max_payload} + {UDPS_HEADER_SIZE}B header exceeds the "
f"{MAX_UDP_PAYLOAD}B UDP datagram limit (sendto EMSGSIZE); "
f"cap at {MAX_UDP_PAYLOAD - UDPS_HEADER_SIZE}")
hub = s.get("hub")
if hub:
ids = [x["id"] for x in s.get("sources", [])]
if s["network"] != "tcp_service":
errs.append("hub mux/join needs network=tcp_service")
elif hub.get("mode") == "mux":
if not all(i.startswith(hub["node"] + "_") for i in ids):
errs.append(f"mux source ids must start with '{hub['node']}_'")
elif hub.get("mode") == "join":
if hub.get("master") not in ids or hub.get("name") in ids:
errs.append("join master must be a source id and the name must differ from all of them")
if s["publishing"] == "Accumulate":
errs.append("join cannot merge Accumulate sources")
else:
errs.append("hub.mode must be mux|join")
for src in s.get("sources", []):
if s["network"] == "multicast":
if not src.get("multicast_group") or not src.get("data_port"):
@@ -250,7 +288,7 @@ def _src(sid, signals, multicast=False):
def mk(sid, desc, sources, network="unicast", publishing="Strict",
ratio=None, min_refresh_hz=None, max_payload=1400,
oracle="analytic", checks=("live", "zoom"), trig=None,
known_issue=None, row_dt=None, num_rows=None, producer_hz=None):
known_issue=None, row_dt=None, num_rows=None, producer_hz=None, hub=None):
# row_dt / num_rows / producer_hz override the global producer geometry for
# this scenario only (default None == use the suite-wide NUM_ROWS / ROW_DT /
# 1 kHz). They are kept together so the per-cycle wall gap (1/producer_hz),
@@ -264,6 +302,7 @@ def mk(sid, desc, sources, network="unicast", publishing="Strict",
"client_checks": list(checks), "trig_signal": trig,
"known_issue": known_issue,
"row_dt": row_dt, "num_rows": num_rows, "producer_hz": producer_hz,
"hub": hub,
}
@@ -577,9 +616,94 @@ _HIGHRATE = [
checks=("live", "zoom")),
]
# ── TCP transports ────────────────────────────────────────────────────────────
# Same chain, TCPStreamer instead of UDPStreamer. "tcp": one listening socket per
# source. "tcp_service": one TCPStreamService (in-process, root object) serves all
# sources; each is a TCPStreamer joined with Service/StreamName and the hub
# subscribes to it by name. TCP does not fragment (max_payload only sizes
# Accumulate batches), so the UDP datagram cap does not apply.
_TCP = [
mk("s60_tcp_scalar", "TCPStreamer: float32 sine 5 Hz + uint32 counter, all checks",
[_src("src", [_sig("Counter", "uint32", 1, formula="counter"),
_sig("Sine", "float32", 1, formula="sine", freq=5.0, unit="V")])],
network="tcp", checks=("live", "zoom", "window", "trigger"), trig="src:Sine"),
mk("s61_tcp_arr_fullarray", "TCPStreamer: 32-elem float32 FullArray sine 5 Hz",
[_src("src", [
_sig("TimeArr", "uint64", 32, unit="ns", formula="time_ns", is_time=True),
_sig("Wave", "float32", 32, time_mode="FullArray",
time_signal="TimeArr", formula="sine", freq=5.0),
])],
network="tcp", oracle="both"),
mk("s62_tcp_accumulate", "TCPStreamer: Accumulate @100 Hz scalar sine 5 Hz",
[_src("src", [_sig("Sine", "float32", 1, formula="sine", freq=5.0)])],
network="tcp", publishing="Accumulate", min_refresh_hz=100.0),
mk("s63_tcp_decimate_quant", "TCPStreamer: Decimate 2 + 16-elem quant uint16 sine 5 Hz",
[_src("src", [_sig("Wave", "float32", 16, quant="uint16", range_min=-1.0,
range_max=1.0, formula="sine", freq=5.0)])],
network="tcp", publishing="Decimate", ratio=2),
mk("s64_tcp_multisrc", "TCPStreamer: two sources on separate ports",
[_src("a", [_sig("Sine", "float32", 1, formula="sine", freq=5.0)]),
_src("b", [_sig("Cnt", "uint32", 1, formula="counter")])],
network="tcp"),
mk("s65_tcp_highrate",
"TCPStreamer: 4x float32 2000-elem arrays, FirstSample 200 kSps, 100 Hz "
"(~3 MB/s, 32 KB frames)",
[_src("src", [
_sig("Tns", "uint64", 1, unit="ns", formula="time_ns", is_time=True),
] + [
_sig(f"S{i}", "float32", 2000, time_mode="FirstSample",
time_signal="Tns", sampling_rate=2.0e5, formula="ramp")
for i in range(4)
])],
network="tcp", max_payload=40000, row_dt=0.01, num_rows=50, producer_hz=100,
checks=("live", "zoom")),
mk("s66_svc_scalar", "TCPStreamService: one stream, sine 5 Hz + counter, all checks",
[_src("src", [_sig("Counter", "uint32", 1, formula="counter"),
_sig("Sine", "float32", 1, formula="sine", freq=5.0, unit="V")])],
network="tcp_service", checks=("live", "zoom", "window", "trigger"),
trig="src:Sine"),
mk("s67_svc_multisrc",
"TCPStreamService: three streams of one service (int16 ramp / float64 sine / uint8 counter)",
[_src("a", [_sig("Ramp", "int16", 1, formula="ramp")]),
_src("b", [_sig("Sine", "float64", 1, formula="sine", freq=5.0)]),
_src("c", [_sig("Cnt", "uint8", 1, formula="counter")])],
network="tcp_service"),
mk("s68_svc_arr_fullarray", "TCPStreamService: 32-elem float32 FullArray sine 5 Hz",
[_src("src", [
_sig("TimeArr", "uint64", 32, unit="ns", formula="time_ns", is_time=True),
_sig("Wave", "float32", 32, time_mode="FullArray",
time_signal="TimeArr", formula="sine", freq=5.0),
])],
network="tcp_service", oracle="both"),
mk("s69_svc_accumulate", "TCPStreamService: Accumulate @100 Hz, two streams",
[_src("a", [_sig("Sine", "float32", 1, formula="sine", freq=5.0)]),
_src("b", [_sig("Sine", "float32", 1, formula="sine", freq=10.0)])],
network="tcp_service", publishing="Accumulate", min_refresh_hz=100.0),
mk("s70_svc_decimate_quant", "TCPStreamService: Decimate 2 + 16-elem quant uint16 sine 5 Hz",
[_src("src", [_sig("Wave", "float32", 16, quant="uint16", range_min=-1.0,
range_max=1.0, formula="sine", freq=5.0)])],
network="tcp_service", publishing="Decimate", ratio=2),
mk("s71_svc_six_streams",
"TCPStreamService: six concurrent streams share two worker threads",
[_src(f"s{i}", [_sig("Sine", "float32", 1, formula="sine", freq=5.0 * (i + 1))])
for i in range(6)],
network="tcp_service"),
mk("s72_svc_mux",
"TCPStreamService: hub takes three streams over ONE multiplexed connection",
[_src("mux_a", [_sig("Ramp", "int16", 1, formula="ramp")]),
_src("mux_b", [_sig("Sine", "float64", 1, formula="sine", freq=5.0)]),
_src("mux_c", [_sig("Cnt", "uint8", 1, formula="counter")])],
network="tcp_service", hub={"mode": "mux", "node": "mux"}),
mk("s73_svc_join",
"TCPStreamJoin: two streams merged into one (master b), hub subscribes to the merge",
[_src("a", [_sig("Counter", "uint32", 1, formula="counter")]),
_src("b", [_sig("Sine", "float32", 1, formula="sine", freq=5.0, unit="V")])],
network="tcp_service", hub={"mode": "join", "name": "all", "master": "b"}),
]
SCENARIOS = (_STARTERS + _TYPES + _ARRAYS + _TIMEMODES + _QUANT + _PUBLISH +
_PAYLOAD + _MCAST + _MULTISRC + _INTERACT + _TRIGGER + _MISC +
_HIGHRATE)
_HIGHRATE + _TCP)
for _s in SCENARIOS:
_s.setdefault("kind", "chain")
@@ -606,6 +730,23 @@ _DIRECT = [
"client_checks": [],
"known_issue": None,
},
{
"id": "s58_direct_tcp",
"desc": "Direct TCPStreamer->TCPStreamerClient round-trip",
"kind": "direct",
"cfg": "Test/E2E/datasources/E2ETCPTest.cfg",
"client_checks": [],
"known_issue": None,
},
{
"id": "s59_direct_tcp_service",
"desc": "Direct TCPStreamer (in-process TCPStreamService) -> TCPStreamerClient "
"(named-stream subscription) round-trip",
"kind": "direct",
"cfg": "Test/E2E/datasources/E2ETCPServiceTest.cfg",
"client_checks": [],
"known_issue": None,
},
]
_RECORDER = [