feat(tcp): TCPStreamer DataSources, in-process TCPStreamService, stream multiplexing and TCPStreamJoin

Add TCPStreamer/TCPStreamerClient (TCP transport behind the UDPSServerI/UDPSClientI
seam), an optional single in-process TCPStreamService serving many streams over
worker threads, multiplexed SUBSCRIBE (comma list, stream index in fragmentIdx),
and TCPStreamJoin, which merges several streams into one sample-and-hold stream
paced by a master. StreamHub takes a comma list of streams over one connection.
Includes GTests, E2E scenarios s58-s73 and docs.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Martino FerrariandClaude Opus 4.6 committed 2026-11-06 13:41:27 +01:00
1 parent 5562877c99
commit 50d85363a3
79 files changed
+12348 -138

No files matched your search

+203 -9
View File
@@ -92,6 +92,7 @@ StreamHub::StreamHub()
memset(calibration_, 0, sizeof(CalibrationEntry) * kMaxCalibration);
for (uint32 i = 0u; i < kMaxSessions; i++) {
sessionActive_[i] = false;
sessionMux_[i] = -1;
configBroadcast_[i] = false;
capHarvested_[i] = false;
for (uint32 s = 0u; s < UDPSS_MAX_SIGNALS; s++) {
@@ -273,11 +274,33 @@ bool StreamHub::Initialise(StructuredDataI &cfg) {
}
cfg.Read("DataPort", dataPort32);
char transport[8] = "udp";
char streamName[256] = "";
StreamString trStr;
if (cfg.Read("Transport", trStr)) {
strncpy(transport, trStr.Buffer(), sizeof(transport) - 1u);
}
StreamString snStr;
if (cfg.Read("StreamName", snStr)) {
strncpy(streamName, snStr.Buffer(), sizeof(streamName) - 1u);
}
if ((strcmp(transport, "tcp") == 0) && (strchr(streamName, ',') != static_cast<char *>(0))) {
/* Several streams over one connection: slots are allocated by the helper. */
if (!AddMuxSourceInternal(label, addr, port32, addr, streamName, nodeId.Buffer())) {
REPORT_ERROR_STATIC(MARTe::ErrorManagement::Warning,
"StreamHub: failed to start multiplexed source '%s'.", nodeId.Buffer());
}
cfg.MoveToAncestor(1u);
continue;
}
sessions_[numSessions_].SetRingCapacities(ringTemporal_, ringScalar_);
bool ok = sessions_[numSessions_].Initialise(
nodeId.Buffer(), label, addr,
static_cast<uint16>(port32), maxPoints_,
mcGroup, static_cast<uint16>(dataPort32));
mcGroup, static_cast<uint16>(dataPort32),
transport, streamName);
if (ok) {
sessions_[numSessions_].SetRecorderConfig(recorderCfg_);
@@ -1201,15 +1224,20 @@ void StreamHub::HandleAddSource(const char *json) {
JsonGetString(json, "addr", addr, sizeof(addr));
JsonGetString(json, "multicastGroup", mcGroup, sizeof(mcGroup));
JsonGetFloat(json, "dataPort", dataPortF);
char transport[8] = "udp";
char streamName[256] = "";
JsonGetString(json, "transport", transport, sizeof(transport));
JsonGetString(json, "streamName", streamName, sizeof(streamName));
if (AddSourceInternal(label, addr, mcGroup,
static_cast<uint16>(dataPortF))) {
static_cast<uint16>(dataPortF), transport, streamName)) {
BroadcastSources();
}
}
bool StreamHub::AddSourceInternal(const char *label, const char *addrPort,
const char *mcGroup, uint16 dataPort) {
const char *mcGroup, uint16 dataPort,
const char *transport, const char *streamName) {
if ((addrPort == static_cast<const char *>(0)) || (addrPort[0] == '\0')) {
return false;
}
@@ -1226,6 +1254,12 @@ bool StreamHub::AddSourceInternal(const char *label, const char *addrPort,
*colon = '\0';
}
if ((transport != static_cast<const char *>(0)) && (strcmp(transport, "tcp") == 0) &&
(streamName != static_cast<const char *>(0)) &&
(strchr(streamName, ',') != static_cast<const char *>(0))) {
return AddMuxSourceInternal(label, host, port, addrPort, streamName);
}
/* Whole add is serialized against other WS-thread add/remove requests.
* The push thread never takes this mutex: it only observes the
* sessionActive_[] flag, which is set last (after a successful Start). */
@@ -1256,7 +1290,7 @@ bool StreamHub::AddSourceInternal(const char *label, const char *addrPort,
sessions_[idx].SetRingCapacities(ringTemporal_, ringScalar_);
bool ok = sessions_[idx].Initialise(id, lbl, host,
static_cast<uint16>(port), maxPoints_,
mcGroup, dataPort);
mcGroup, dataPort, transport, streamName);
if (ok) {
sessions_[idx].SetRecorderConfig(recorderCfg_);
ok = sessions_[idx].Start();
@@ -1275,14 +1309,151 @@ bool StreamHub::AddSourceInternal(const char *label, const char *addrPort,
return ok;
}
bool StreamHub::SourceIsActive(const char *addrPort) {
bool StreamHub::AddMuxSourceInternal(const char *label, const char *host, uint32 port,
const char *addrPort, const char *streamList,
const char *idPrefix) {
char list[MARTe::UDPS_TCP_MAX_SUBSCRIBE_PAYLOAD + 1u];
strncpy(list, streamList, sizeof(list) - 1u);
list[sizeof(list) - 1u] = '\0';
const char *names[kMaxSessions];
uint32 n = 0u;
char *cur = list;
while (cur != static_cast<char *>(0)) {
char *comma = strchr(cur, ',');
if (comma != static_cast<char *>(0)) { *comma = '\0'; }
if ((cur[0] == '\0') || (n >= kMaxSessions)) {
REPORT_ERROR_STATIC(MARTe::ErrorManagement::Warning,
"StreamHub: bad stream list '%s' (empty name or more than %u streams).",
streamList, kMaxSessions);
return false;
}
names[n++] = cur;
cur = (comma != static_cast<char *>(0)) ? (comma + 1) : static_cast<char *>(0);
}
(void) sessionsMutex_.FastLock();
uint32 slots[kMaxSessions];
uint32 found = 0u;
for (uint32 i = 0u; (i < kMaxSessions) && (found < n); i++) {
if (!sessionActive_[i]) { slots[found++] = i; }
}
uint32 mux = kMaxSessions;
for (uint32 i = 0u; i < kMaxSessions; i++) {
if (muxClients_[i].client == static_cast<MARTe::TCPSClient *>(0)) { mux = i; break; }
}
if ((found < n) || (mux == kMaxSessions)) {
sessionsMutex_.FastUnLock();
REPORT_ERROR_STATIC(MARTe::ErrorManagement::Warning,
"StreamHub: not enough free sessions for %u streams, ignoring addSource.", n);
return false;
}
MARTe::TCPSClient *client = new MARTe::TCPSClient();
ConfigurationDatabase cfg;
(void) cfg.Write("ServerAddr", host);
(void) cfg.Write("Port", port);
(void) cfg.Write("StreamName", streamList);
bool ok = client->Initialise(cfg) && (client->GetNumStreams() == n);
if (!ok) {
delete client;
sessionsMutex_.FastUnLock();
REPORT_ERROR_STATIC(MARTe::ErrorManagement::Warning,
"StreamHub: could not set up the multiplexed receiver for '%s'.", streamList);
return false;
}
muxClients_[mux].client = client;
muxClients_[mux].refs = 0u;
const char *base = ((label != static_cast<const char *>(0)) && (label[0] != '\0')) ? label
: static_cast<const char *>(0);
/* Initialise every session (listeners attached) before any can receive. */
uint32 initialised = 0u;
for (uint32 k = 0u; (k < n) && ok; k++) {
const uint32 idx = slots[k];
char id[16 + 256];
if (idPrefix != static_cast<const char *>(0)) {
snprintf(id, sizeof(id), "%s_%s", idPrefix, names[k]);
} else {
snprintf(id, sizeof(id), "s%u", nextSourceId_++);
}
char lbl[160];
if (base != static_cast<const char *>(0)) {
snprintf(lbl, sizeof(lbl), "%s_%s", base, names[k]);
} else {
snprintf(lbl, sizeof(lbl), "%s", names[k]);
}
configBroadcast_[idx] = false;
for (uint32 s = 0u; s < UDPSS_MAX_SIGNALS; s++) { pushCursor_[idx][s] = 0u; }
sessions_[idx].SetRingCapacities(ringTemporal_, ringScalar_);
ok = sessions_[idx].Initialise(id, lbl, host, static_cast<uint16>(port), maxPoints_,
"", 0u, "tcp", names[k], client->GetStream(k));
if (ok) {
sessionMux_[idx] = static_cast<MARTe::int32>(mux);
muxClients_[mux].refs++;
initialised++;
}
}
for (uint32 k = 0u; (k < initialised) && ok; k++) {
sessions_[slots[k]].SetRecorderConfig(recorderCfg_);
ok = sessions_[slots[k]].Start();
}
if (ok) {
for (uint32 k = 0u; k < n; k++) {
sessionActive_[slots[k]] = true;
numSessions_++;
}
} else {
for (uint32 k = 0u; k < initialised; k++) { ReleaseMuxSlot(slots[k]); }
if (muxClients_[mux].client != static_cast<MARTe::TCPSClient *>(0)) {
delete muxClients_[mux].client; /* no session ever attached */
muxClients_[mux].client = static_cast<MARTe::TCPSClient *>(0);
}
}
sessionsMutex_.FastUnLock();
if (ok) {
REPORT_ERROR_STATIC(MARTe::ErrorManagement::Information,
"StreamHub: added %u multiplexed sessions over one connection (%s).", n, addrPort);
}
return ok;
}
void StreamHub::ReleaseMuxSlot(uint32 slot) {
/* Caller holds sessionsMutex_. */
if (sessionMux_[slot] < 0) { return; }
const uint32 m = static_cast<uint32>(sessionMux_[slot]);
sessions_[slot].DetachClient();
sessionMux_[slot] = -1;
if ((muxClients_[m].refs > 0u) && (--muxClients_[m].refs == 0u)) {
delete muxClients_[m].client;
muxClients_[m].client = static_cast<MARTe::TCPSClient *>(0);
}
}
bool StreamHub::SourceIsActive(const char *addrPort, const char *streamName) {
if (strchr(streamName, ',') != static_cast<const char *>(0)) {
/* A list is active when every stream of it is. */
char list[MARTe::UDPS_TCP_MAX_SUBSCRIBE_PAYLOAD + 1u];
strncpy(list, streamName, sizeof(list) - 1u);
list[sizeof(list) - 1u] = '\0';
bool all = true;
char *cur = list;
while (all && (cur != static_cast<char *>(0))) {
char *comma = strchr(cur, ',');
if (comma != static_cast<char *>(0)) { *comma = '\0'; }
all = SourceIsActive(addrPort, cur);
cur = (comma != static_cast<char *>(0)) ? (comma + 1) : static_cast<char *>(0);
}
return all;
}
for (uint32 i = 0u; i < kMaxSessions; i++) {
if (!sessionActive_[i]) { continue; }
StreamString adr = sessions_[i].GetAddr();
char cur[96];
(void) snprintf(cur, sizeof(cur), "%s:%u", adr.Buffer(),
static_cast<uint32>(sessions_[i].GetPort()));
if (strcmp(cur, addrPort) == 0) { return true; }
StreamString sn = sessions_[i].GetStreamName();
if ((strcmp(cur, addrPort) == 0) && (strcmp(sn.Buffer(), streamName) == 0)) { return true; }
}
return false;
}
@@ -1335,11 +1506,16 @@ bool StreamHub::LoadSourcesFile(bool skipActive, bool clearCalibration) {
(void) JsonGetString(obj, "label", label, sizeof(label));
(void) JsonGetString(obj, "multicastGroup", mcGroup, sizeof(mcGroup));
(void) JsonGetFloat(obj, "dataPort", dataPortF);
if (skipActive && SourceIsActive(addr)) {
char transport[8] = "udp";
char streamName[256] = "";
(void) JsonGetString(obj, "transport", transport, sizeof(transport));
(void) JsonGetString(obj, "streamName", streamName, sizeof(streamName));
if (skipActive && SourceIsActive(addr, streamName)) {
/* Already streaming — leave the live session untouched. */
}
else if (AddSourceInternal(label, addr, mcGroup,
static_cast<uint16>(dataPortF))) {
static_cast<uint16>(dataPortF),
transport, streamName)) {
nLoaded++;
}
}
@@ -1414,6 +1590,14 @@ void StreamHub::HandleSaveSources() {
static_cast<uint32>(dpt));
}
}
StreamString trn = sessions_[i].GetTransport();
if (strcmp(trn.Buffer(), "tcp") == 0) {
(void) fprintf(f, ",\n \"transport\": \"tcp\"");
StreamString sn = sessions_[i].GetStreamName();
if (sn.Size() > 0u) {
(void) fprintf(f, ",\n \"streamName\": \"%s\"", sn.Buffer());
}
}
(void) fprintf(f, "\n }");
nSaved++;
}
@@ -1500,16 +1684,26 @@ void StreamHub::HandleRemoveSource(const char *json) {
break;
}
}
bool shared = false;
if (found < kMaxSessions) {
sessionActive_[found] = false;
configBroadcast_[found] = false; /* allow re-broadcast on slot reuse */
numSessions_--;
if (sessionMux_[found] >= 0) {
/* One stream of a shared receiver: stop it and drop its listener
* under the lock, so the slot cannot be reused underneath. */
sessions_[found].Stop();
ReleaseMuxSlot(found);
shared = true;
}
}
sessionsMutex_.FastUnLock();
if (found == kMaxSessions) { return; }
sessions_[found].Stop();
if (!shared) {
sessions_[found].Stop();
}
REPORT_ERROR_STATIC(MARTe::ErrorManagement::Information,
"StreamHub: removed session '%s'.", id);
+34 -3
View File
@@ -231,11 +231,27 @@ private:
* @return true if the session was started.
*/
bool AddSourceInternal(const char *label, const char *addrPort,
const char *mcGroup, uint16 dataPort);
const char *mcGroup, uint16 dataPort,
const char *transport = "udp",
const char *streamName = "");
/**
* @brief TCP source whose streamName is a comma separated list "a,b,c": one
* TCPSClient (one socket) feeds one session per stream, labelled
* "<label>_<stream>" (just "<stream>" without a label). Session ids are
* "s<N>", or "<idPrefix>_<stream>" when @p idPrefix (the config node id) is given.
* @return true if every session was started (none is left behind otherwise).
*/
bool AddMuxSourceInternal(const char *label, const char *host, uint32 port,
const char *addrPort, const char *streamList,
const char *idPrefix = static_cast<const char *>(0));
/** @brief Detach @p slot from its shared receiver; frees the receiver with its last session. */
void ReleaseMuxSlot(uint32 slot);
/**
* @brief Load sources and calibration from sourcesFile_ (a flat JSON array
* of {"label","addr","multicastGroup","dataPort"} source blocks and
* of {"label","addr","multicastGroup","dataPort","transport","streamName"} source blocks and
* {"source","signal","scale","offset","unit"} calibration blocks).
* @param skipActive when true, a source whose "host:port" is already
* streaming is left alone instead of being started a second time.
@@ -247,7 +263,7 @@ private:
bool LoadSourcesFile(bool skipActive, bool clearCalibration = false);
/** @return true if a session for this "host:port" is already active. */
bool SourceIsActive(const char *addrPort);
bool SourceIsActive(const char *addrPort, const char *streamName = "");
/**
* @brief Store or replace one calibration entry. An identity entry
@@ -280,6 +296,21 @@ private:
/* ---- Members --------------------------------------------------------- */
/** A multiplexed receiver shared by several sessions (deleted with the last one). */
struct MuxClient {
MuxClient() : client(static_cast<MARTe::TCPSClient *>(0)), refs(0u) {}
~MuxClient() { delete client; }
MARTe::TCPSClient *client;
uint32 refs;
private:
MuxClient(const MuxClient &);
MuxClient &operator=(const MuxClient &);
};
/* Declared before sessions_ so it is destroyed after them: a session stops
* its (facade of a) shared receiver from its own destructor. */
MuxClient muxClients_[kMaxSessions];
MARTe::int32 sessionMux_[kMaxSessions]; ///< Index into muxClients_, or -1
UDPSourceSession sessions_[kMaxSessions];
uint32 numSessions_; ///< Count of active sessions (not an upper slot bound)
bool sessionActive_[kMaxSessions]; ///< Per-slot in-use flag (slots are not compacted on removal)
@@ -51,6 +51,8 @@ UDPSourceSession::UDPSourceSession()
ringTemporal_(1000000u),
ringScalar_(100000u),
initialised_(false),
client_(static_cast<MARTe::UDPSClientI *>(0)),
external_(false),
numSignals_(0u),
publishMode_(0u),
configured_(false),
@@ -112,7 +114,10 @@ bool UDPSourceSession::Initialise(const char *id, const char *label,
const char *addr, uint16 port,
uint32 maxPts,
const char *mcGroup,
uint16 dataPort) {
uint16 dataPort,
const char *transport,
const char *streamName,
MARTe::UDPSClientI *external) {
(void) Stop();
id_ = id;
@@ -121,25 +126,42 @@ bool UDPSourceSession::Initialise(const char *id, const char *label,
port_ = port;
mcGroup_ = (mcGroup != static_cast<const char *>(0)) ? mcGroup : "";
dataPort_ = dataPort;
transport_ = ((transport != static_cast<const char *>(0)) && (strcmp(transport, "tcp") == 0)) ? "tcp" : "udp";
streamName_ = (streamName != static_cast<const char *>(0)) ? streamName : "";
maxPoints_ = (maxPts > 0u) ? maxPts : 20000u;
/* Build ConfigurationDatabase for UDPSClient */
ConfigurationDatabase cfg;
(void) cfg.Write("ServerAddr", addr);
(void) cfg.Write("Port", static_cast<uint32>(port));
if ((mcGroup != static_cast<const char *>(0)) && (strlen(mcGroup) > 0u)) {
const bool useTcp = (transport_ == "tcp");
if (useTcp) {
if (streamName_.Size() > 0u) {
(void) cfg.Write("StreamName", streamName_.Buffer());
}
} else if ((mcGroup != static_cast<const char *>(0)) && (strlen(mcGroup) > 0u)) {
(void) cfg.Write("MulticastGroup", mcGroup);
if (dataPort > 0u) {
(void) cfg.Write("DataPort", static_cast<uint32>(dataPort));
}
}
if (!client_.Initialise(cfg)) {
external_ = (external != static_cast<MARTe::UDPSClientI *>(0));
if (external_) {
client_ = external; /* already configured by its owner */
}
else {
client_ = useTcp ? static_cast<MARTe::UDPSClientI *>(&tcpClient_)
: static_cast<MARTe::UDPSClientI *>(&udpClient_);
}
if (!external_ && !client_->Initialise(cfg)) {
REPORT_ERROR_STATIC(MARTe::ErrorManagement::FatalError,
"UDPSourceSession[%s]: Failed to initialise UDPSClient.", id);
"UDPSourceSession[%s]: Failed to initialise %s client.",
id, useTcp ? "TCPSClient" : "UDPSClient");
client_ = static_cast<MARTe::UDPSClientI *>(0);
return false;
}
client_.SetListener(this);
client_->SetListener(this);
(void) metaMutex_.FastLock();
numSignals_ = 0u;
@@ -165,12 +187,22 @@ bool UDPSourceSession::Start() {
statByteCount_ = 0u;
stateStr_ = "connecting";
statsMutex_.FastUnLock();
return client_.Start();
return client_->Start();
}
void UDPSourceSession::DetachClient() {
if (!external_) { return; }
(void) Stop();
if (client_ != static_cast<MARTe::UDPSClientI *>(0)) {
client_->SetListener(static_cast<MARTe::UDPSClientListener *>(0));
}
client_ = static_cast<MARTe::UDPSClientI *>(0);
external_ = false;
}
bool UDPSourceSession::Stop() {
if (!initialised_) { return true; }
return client_.Stop();
if (!initialised_ || (client_ == static_cast<MARTe::UDPSClientI *>(0))) { return true; }
return client_->Stop();
}
/*---------------------------------------------------------------------------*/
@@ -200,7 +232,7 @@ void UDPSourceSession::OnUDPSConfig(const uint8 *payload, uint32 payloadSize) {
void UDPSourceSession::OnUDPSData(const uint8 *payload, uint32 payloadSize) {
/* Valid only for the duration of this callback. */
ParseDataPayload(payload, payloadSize, client_.GetLastDataGap());
ParseDataPayload(payload, payloadSize, client_->GetLastDataGap());
}
/*---------------------------------------------------------------------------*/
@@ -947,6 +979,10 @@ StreamString UDPSourceSession::GetMulticastGroup() const { return mcGroup_; }
uint16 UDPSourceSession::GetDataPort() const { return dataPort_; }
StreamString UDPSourceSession::GetTransport() const { return transport_; }
StreamString UDPSourceSession::GetStreamName() const { return streamName_; }
bool UDPSourceSession::IsConfigured() const {
(void) metaMutex_.FastLock();
bool c = configured_;
@@ -11,6 +11,7 @@
#define STREAMHUB_UDP_SOURCE_SESSION_H_
#include "UDPSClient.h"
#include "TCPSClient.h"
#include "UDPSProtocol.h"
#include "FastPollingMutexSem.h"
#include "HighResolutionTimer.h"
@@ -158,11 +159,23 @@ public:
* @param maxPts Ring buffer capacity per signal.
* @param mcGroup Multicast group (empty string = unicast).
* @param dataPort Multicast data port (0 = port+1).
* @param transport "udp" (default) or "tcp". TCP ignores mcGroup/dataPort.
* @param streamName TCP only: stream to subscribe to on a TCPStreamService
* ("" = plain TCPStreamer, no subscription).
* @param external Optional receiver owned by the caller (one stream of a
* multiplexed TCPSClient, see TCPSClient::GetStream()). When set the
* session does not create or configure its own client; call
* DetachClient() before the receiver is destroyed.
* @return true on success.
*/
bool Initialise(const char *id, const char *label,
const char *addr, uint16 port, uint32 maxPts,
const char *mcGroup = "", uint16 dataPort = 0u);
const char *mcGroup = "", uint16 dataPort = 0u,
const char *transport = "udp", const char *streamName = "",
MARTe::UDPSClientI *external = static_cast<MARTe::UDPSClientI *>(0));
/** @brief Stop and forget an external receiver (no-op for built-in receivers). */
void DetachClient();
/** @brief Start the UDPSClient background thread. */
bool Start();
@@ -236,6 +249,12 @@ public:
/** @return Multicast data port (0 = port+1 default). */
uint16 GetDataPort() const;
/** @return "udp" or "tcp". */
StreamString GetTransport() const;
/** @return TCP stream name ("" = none). */
StreamString GetStreamName() const;
/** @return true after the first CONFIG packet has been fully parsed. */
bool IsConfigured() const;
@@ -443,13 +462,18 @@ private:
uint16 port_;
StreamString mcGroup_; ///< Multicast group ("" = unicast)
uint16 dataPort_; ///< Multicast data port (0 = port+1)
StreamString transport_; ///< "udp" or "tcp"
StreamString streamName_;///< TCPStreamService stream name ("" = none)
uint32 maxPoints_;
uint32 ringTemporal_; ///< Ring capacity for multi-element signals
uint32 ringScalar_; ///< Ring capacity for scalar signals
bool initialised_;
/* UDPSClient (owns the background thread) */
UDPSClient client_;
/* Receivers (each owns a background thread); client_ points at the active one. */
UDPSClient udpClient_;
MARTe::TCPSClient tcpClient_;
MARTe::UDPSClientI *client_;
bool external_; ///< client_ is owned by the caller
/* Signal metadata — protected by metaMutex_ */
mutable FastPollingMutexSem metaMutex_;