feat(streamhub): BinaryRecorder core (header, encode, rows, flush, rotation)
Add a FileWriter-compatible per-source binary recorder: native-type encode and TypeDescriptor mapping, FileWriter header serialization, subset/quantized/ ACCUMULATE row serialization, and push-thread double-buffer flush with size-cap rotation, keep-N pruning, fdatasync cadence, disk-free and staging-overflow guards. Covered by 12 GTests; full suite (83) green. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,565 @@
|
||||
/**
|
||||
* @file BinaryRecorder.cpp
|
||||
* @brief Per-source binary recorder implementation.
|
||||
*
|
||||
* @copyright Copyright 2015 F4E | European Joint Undertaking for ITER and
|
||||
* the Development of Fusion Energy ('Fusion for Energy').
|
||||
* Licensed under the EUPL, Version 1.1 or - as soon they will be approved
|
||||
* by the European Commission - subsequent versions of the EUPL (the "Licence")
|
||||
* You may not use this work except in compliance with the Licence.
|
||||
* You may obtain a copy of the Licence at: http://ec.europa.eu/idabc/eupl
|
||||
*
|
||||
* @warning Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the Licence is distributed on an "AS IS"
|
||||
* basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
* or implied. See the Licence permissions and limitations under the Licence.
|
||||
*/
|
||||
|
||||
#include "BinaryRecorder.h"
|
||||
|
||||
#include "AdvancedErrorManagement.h"
|
||||
#include "TypeDescriptor.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <time.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/statvfs.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#include <dirent.h>
|
||||
|
||||
namespace StreamHub {
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
/* Static helpers */
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
uint16 BinaryRecorder::TypeCodeToDescriptorAll(uint8 udpsTypeCode) {
|
||||
switch (udpsTypeCode) {
|
||||
case MARTe::UDPS_TYPECODE_UINT8: return MARTe::UnsignedInteger8Bit.all;
|
||||
case MARTe::UDPS_TYPECODE_INT8: return MARTe::SignedInteger8Bit.all;
|
||||
case MARTe::UDPS_TYPECODE_UINT16: return MARTe::UnsignedInteger16Bit.all;
|
||||
case MARTe::UDPS_TYPECODE_INT16: return MARTe::SignedInteger16Bit.all;
|
||||
case MARTe::UDPS_TYPECODE_UINT32: return MARTe::UnsignedInteger32Bit.all;
|
||||
case MARTe::UDPS_TYPECODE_INT32: return MARTe::SignedInteger32Bit.all;
|
||||
case MARTe::UDPS_TYPECODE_UINT64: return MARTe::UnsignedInteger64Bit.all;
|
||||
case MARTe::UDPS_TYPECODE_INT64: return MARTe::SignedInteger64Bit.all;
|
||||
case MARTe::UDPS_TYPECODE_FLOAT32: return MARTe::Float32Bit.all;
|
||||
case MARTe::UDPS_TYPECODE_FLOAT64: return MARTe::Float64Bit.all;
|
||||
default: return MARTe::Float64Bit.all;
|
||||
}
|
||||
}
|
||||
|
||||
uint32 BinaryRecorder::EncodeNative(uint8 typeCode, float64 v, uint8 *dst) {
|
||||
switch (typeCode) {
|
||||
case MARTe::UDPS_TYPECODE_UINT8: {
|
||||
uint8 x = static_cast<uint8>(v); memcpy(dst, &x, 1u); return 1u;
|
||||
}
|
||||
case MARTe::UDPS_TYPECODE_INT8: {
|
||||
MARTe::int8 x = static_cast<MARTe::int8>(v); memcpy(dst, &x, 1u); return 1u;
|
||||
}
|
||||
case MARTe::UDPS_TYPECODE_UINT16: {
|
||||
uint16 x = static_cast<uint16>(v); memcpy(dst, &x, 2u); return 2u;
|
||||
}
|
||||
case MARTe::UDPS_TYPECODE_INT16: {
|
||||
MARTe::int16 x = static_cast<MARTe::int16>(v); memcpy(dst, &x, 2u); return 2u;
|
||||
}
|
||||
case MARTe::UDPS_TYPECODE_UINT32: {
|
||||
uint32 x = static_cast<uint32>(v); memcpy(dst, &x, 4u); return 4u;
|
||||
}
|
||||
case MARTe::UDPS_TYPECODE_INT32: {
|
||||
MARTe::int32 x = static_cast<MARTe::int32>(v); memcpy(dst, &x, 4u); return 4u;
|
||||
}
|
||||
case MARTe::UDPS_TYPECODE_UINT64: {
|
||||
uint64 x = static_cast<uint64>(v); memcpy(dst, &x, 8u); return 8u;
|
||||
}
|
||||
case MARTe::UDPS_TYPECODE_INT64: {
|
||||
MARTe::int64 x = static_cast<MARTe::int64>(v); memcpy(dst, &x, 8u); return 8u;
|
||||
}
|
||||
case MARTe::UDPS_TYPECODE_FLOAT32: {
|
||||
float x = static_cast<float>(v); memcpy(dst, &x, 4u); return 4u;
|
||||
}
|
||||
case MARTe::UDPS_TYPECODE_FLOAT64: {
|
||||
float64 x = v; memcpy(dst, &x, 8u); return 8u;
|
||||
}
|
||||
default: return 0u;
|
||||
}
|
||||
}
|
||||
|
||||
/** Local dequantization mirror of UDPSourceSession::DequantizeValue. */
|
||||
static float64 RecDequantize(float64 q, uint8 quantType,
|
||||
float64 rangeMin, float64 rangeMax) {
|
||||
const float64 range = rangeMax - rangeMin;
|
||||
switch (quantType) {
|
||||
case MARTe::UDPS_QUANT_UINT8: return rangeMin + (q / 255.0) * range;
|
||||
case MARTe::UDPS_QUANT_INT8: return rangeMin + ((q + 127.0) / 254.0) * range;
|
||||
case MARTe::UDPS_QUANT_UINT16: return rangeMin + (q / 65535.0) * range;
|
||||
case MARTe::UDPS_QUANT_INT16: return rangeMin + ((q + 32767.0) / 65534.0) * range;
|
||||
default: return q;
|
||||
}
|
||||
}
|
||||
|
||||
/** Read one quantized wire element as a float64 integer value. */
|
||||
static float64 RecDecodeQuantWire(const uint8 *ptr, uint8 quantType) {
|
||||
switch (quantType) {
|
||||
case MARTe::UDPS_QUANT_UINT8: {
|
||||
uint8 v = 0u; memcpy(&v, ptr, 1u); return static_cast<float64>(v);
|
||||
}
|
||||
case MARTe::UDPS_QUANT_INT8: {
|
||||
MARTe::int8 v = 0; memcpy(&v, ptr, 1u); return static_cast<float64>(v);
|
||||
}
|
||||
case MARTe::UDPS_QUANT_UINT16: {
|
||||
uint16 v = 0u; memcpy(&v, ptr, 2u); return static_cast<float64>(v);
|
||||
}
|
||||
case MARTe::UDPS_QUANT_INT16: {
|
||||
MARTe::int16 v = 0; memcpy(&v, ptr, 2u); return static_cast<float64>(v);
|
||||
}
|
||||
default: return 0.0;
|
||||
}
|
||||
}
|
||||
|
||||
static uint32 RecQuantWireBytes(uint8 quantType) {
|
||||
switch (quantType) {
|
||||
case MARTe::UDPS_QUANT_UINT8: return 1u;
|
||||
case MARTe::UDPS_QUANT_INT8: return 1u;
|
||||
case MARTe::UDPS_QUANT_UINT16: return 2u;
|
||||
case MARTe::UDPS_QUANT_INT16: return 2u;
|
||||
default: return 0u;
|
||||
}
|
||||
}
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
/* Constructor / Destructor */
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
BinaryRecorder::BinaryRecorder()
|
||||
: nIncluded_(0u),
|
||||
rowBytes_(0u),
|
||||
headerLen_(0u),
|
||||
layoutReady_(false),
|
||||
pushHeaderLen_(0u),
|
||||
pushRowBytes_(0u),
|
||||
front_(static_cast<uint8 *>(0)),
|
||||
back_(static_cast<uint8 *>(0)),
|
||||
frontLen_(0u),
|
||||
backLen_(0u),
|
||||
pendingArm_(false),
|
||||
pendingDisarm_(false),
|
||||
pendingReopen_(false),
|
||||
fd_(-1),
|
||||
fileOffset_(0u),
|
||||
lastSyncSec_(0u),
|
||||
fileSeq_(0u),
|
||||
armed_(false),
|
||||
diskLow_(false),
|
||||
bytesWritten_(0u),
|
||||
rowsWritten_(0u),
|
||||
droppedRows_(0u) {
|
||||
memset(&config_, 0, sizeof(config_));
|
||||
sourceId_[0] = '\0';
|
||||
curFilePath_[0] = '\0';
|
||||
stagingMutex_.Create();
|
||||
}
|
||||
|
||||
BinaryRecorder::~BinaryRecorder() {
|
||||
CloseFile();
|
||||
if (front_ != static_cast<uint8 *>(0)) { delete[] front_; front_ = static_cast<uint8 *>(0); }
|
||||
if (back_ != static_cast<uint8 *>(0)) { delete[] back_; back_ = static_cast<uint8 *>(0); }
|
||||
}
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
/* Init */
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
void BinaryRecorder::Init(const RecorderConfig &cfg, const char *sourceId) {
|
||||
config_ = cfg;
|
||||
if (config_.stagingBytes < 4096u) { config_.stagingBytes = 4096u; }
|
||||
strncpy(sourceId_, (sourceId != static_cast<const char *>(0)) ? sourceId : "src",
|
||||
sizeof(sourceId_) - 1u);
|
||||
sourceId_[sizeof(sourceId_) - 1u] = '\0';
|
||||
|
||||
front_ = new uint8[config_.stagingBytes];
|
||||
back_ = new uint8[config_.stagingBytes];
|
||||
frontLen_ = 0u;
|
||||
backLen_ = 0u;
|
||||
|
||||
/* Create output directory (best-effort). */
|
||||
if (config_.directory[0] != '\0') {
|
||||
(void) mkdir(config_.directory, 0775);
|
||||
}
|
||||
|
||||
if (config_.autoStart) {
|
||||
pendingArm_ = true;
|
||||
}
|
||||
}
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
/* Configure (receive thread) */
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
void BinaryRecorder::Configure(const MARTe::UDPSSignalDescriptor *descs,
|
||||
uint32 nSigs, const bool *includeMask) {
|
||||
uint32 inc = 0u;
|
||||
for (uint32 s = 0u; (s < nSigs) && (inc < kRecMaxSig); s++) {
|
||||
if ((includeMask != static_cast<const bool *>(0)) && (!includeMask[s])) {
|
||||
continue;
|
||||
}
|
||||
const MARTe::UDPSSignalDescriptor &d = descs[s];
|
||||
uint32 nElems = d.numRows * d.numCols;
|
||||
if (nElems == 0u) { nElems = 1u; }
|
||||
const uint32 native = MARTe::UDPSTypeCodeByteSize(d.typeCode);
|
||||
if (native == 0u) { continue; }
|
||||
const uint32 wire = (d.quantType != MARTe::UDPS_QUANT_NONE)
|
||||
? RecQuantWireBytes(d.quantType)
|
||||
: native;
|
||||
srcIdx_[inc] = s;
|
||||
typeCode_[inc] = d.typeCode;
|
||||
quantType_[inc] = d.quantType;
|
||||
declaredElems_[inc] = nElems;
|
||||
nativeBytes_[inc] = native;
|
||||
wireBytes_[inc] = wire;
|
||||
rangeMin_[inc] = d.rangeMin;
|
||||
rangeMax_[inc] = d.rangeMax;
|
||||
inc++;
|
||||
}
|
||||
nIncluded_ = inc;
|
||||
|
||||
/* Build the FileWriter header blob and compute the row size. */
|
||||
uint32 pos = 0u;
|
||||
memcpy(&headerBlob_[pos], &nIncluded_, 4u); pos += 4u;
|
||||
uint32 row = 0u;
|
||||
for (uint32 i = 0u; i < nIncluded_; i++) {
|
||||
const uint16 tcAll = TypeCodeToDescriptorAll(typeCode_[i]);
|
||||
memcpy(&headerBlob_[pos], &tcAll, 2u); pos += 2u;
|
||||
/* 32-byte name, null-padded, truncated from the descriptor name. */
|
||||
char name32[32];
|
||||
memset(name32, 0, sizeof(name32));
|
||||
strncpy(name32, descs[srcIdx_[i]].name, sizeof(name32) - 1u);
|
||||
memcpy(&headerBlob_[pos], name32, 32u); pos += 32u;
|
||||
memcpy(&headerBlob_[pos], &declaredElems_[i], 4u); pos += 4u;
|
||||
row += declaredElems_[i] * nativeBytes_[i];
|
||||
}
|
||||
headerLen_ = pos;
|
||||
rowBytes_ = row;
|
||||
|
||||
(void) stagingMutex_.FastLock();
|
||||
layoutReady_ = true;
|
||||
pendingReopen_ = true;
|
||||
/* Discard any rows staged under the previous layout: they would belong
|
||||
* to the file being closed and must not be mixed into the new layout. */
|
||||
frontLen_ = 0u;
|
||||
stagingMutex_.FastUnLock();
|
||||
}
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
/* CapturePacket (receive thread) */
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
void BinaryRecorder::CapturePacket(const uint8 *payload, const uint32 *sigOff,
|
||||
const uint32 *sigElems, uint8 publishMode,
|
||||
uint32 numSamples) {
|
||||
if (!layoutReady_ || (nIncluded_ == 0u) || (rowBytes_ == 0u)) { return; }
|
||||
const bool accumulate = (publishMode == MARTe::UDPS_PUBLISH_ACCUMULATE);
|
||||
uint32 numRows = accumulate ? numSamples : 1u;
|
||||
if (numRows == 0u) { numRows = 1u; }
|
||||
|
||||
const uint32 rowSpan = numRows * rowBytes_;
|
||||
|
||||
(void) stagingMutex_.FastLock();
|
||||
if (frontLen_ + rowSpan > config_.stagingBytes) {
|
||||
droppedRows_ += static_cast<uint64>(numRows);
|
||||
stagingMutex_.FastUnLock();
|
||||
return;
|
||||
}
|
||||
|
||||
for (uint32 r = 0u; r < numRows; r++) {
|
||||
for (uint32 i = 0u; i < nIncluded_; i++) {
|
||||
const uint32 s = srcIdx_[i];
|
||||
const uint8 *base = payload + sigOff[s];
|
||||
/* A per-sample (accumulated) scalar carries numSamples wire values;
|
||||
* a plain scalar carries one value that is repeated across rows. */
|
||||
const bool isAccumScalar = accumulate && (declaredElems_[i] == 1u) &&
|
||||
(sigElems[s] > 1u);
|
||||
const uint32 nElem = declaredElems_[i];
|
||||
|
||||
if (quantType_[i] == MARTe::UDPS_QUANT_NONE) {
|
||||
if (isAccumScalar) {
|
||||
/* Take wire element r for an accumulated scalar. */
|
||||
memcpy(&front_[frontLen_], base + r * wireBytes_[i],
|
||||
nativeBytes_[i]);
|
||||
frontLen_ += nativeBytes_[i];
|
||||
}
|
||||
else {
|
||||
/* Straight copy of all elements (wire == native size). */
|
||||
const uint32 bytes = nElem * nativeBytes_[i];
|
||||
memcpy(&front_[frontLen_], base, bytes);
|
||||
frontLen_ += bytes;
|
||||
}
|
||||
}
|
||||
else {
|
||||
/* Quantized: dequantize + re-encode per element. */
|
||||
for (uint32 e = 0u; e < nElem; e++) {
|
||||
const uint32 wireIdx = isAccumScalar ? r : e;
|
||||
const float64 q = RecDecodeQuantWire(
|
||||
base + wireIdx * wireBytes_[i], quantType_[i]);
|
||||
const float64 phys = RecDequantize(q, quantType_[i],
|
||||
rangeMin_[i], rangeMax_[i]);
|
||||
frontLen_ += EncodeNative(typeCode_[i], phys,
|
||||
&front_[frontLen_]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stagingMutex_.FastUnLock();
|
||||
}
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
/* FlushTick (push thread) */
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
void BinaryRecorder::FlushTick(uint32 nowSec) {
|
||||
bool doArm = false;
|
||||
bool doDisarm = false;
|
||||
bool doReopen = false;
|
||||
|
||||
(void) stagingMutex_.FastLock();
|
||||
doArm = pendingArm_; pendingArm_ = false;
|
||||
doDisarm = pendingDisarm_; pendingDisarm_ = false;
|
||||
doReopen = pendingReopen_; pendingReopen_ = false;
|
||||
if (doReopen && layoutReady_) {
|
||||
memcpy(pushHeader_, headerBlob_, headerLen_);
|
||||
pushHeaderLen_ = headerLen_;
|
||||
pushRowBytes_ = rowBytes_;
|
||||
}
|
||||
/* Swap front -> back. */
|
||||
uint8 *tmp = back_;
|
||||
back_ = front_;
|
||||
front_ = tmp;
|
||||
backLen_ = frontLen_;
|
||||
frontLen_ = 0u;
|
||||
stagingMutex_.FastUnLock();
|
||||
|
||||
if (doArm) { armed_ = true; }
|
||||
|
||||
if (doReopen) {
|
||||
/* Layout changed: close the current file. The freshly swapped back
|
||||
* buffer holds new-layout rows and is written to the new file below. */
|
||||
CloseFile();
|
||||
}
|
||||
|
||||
if (doDisarm) {
|
||||
if ((fd_ >= 0) && (backLen_ > 0u)) {
|
||||
(void) pwrite(fd_, back_, backLen_, static_cast<off_t>(fileOffset_));
|
||||
fileOffset_ += backLen_;
|
||||
bytesWritten_ += backLen_;
|
||||
}
|
||||
backLen_ = 0u;
|
||||
armed_ = false;
|
||||
CloseFile();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!armed_) {
|
||||
backLen_ = 0u;
|
||||
return;
|
||||
}
|
||||
|
||||
/* Disk-free guard. */
|
||||
if (!HasSufficientDisk()) {
|
||||
if (!diskLow_) {
|
||||
diskLow_ = true;
|
||||
REPORT_ERROR_STATIC(MARTe::ErrorManagement::Warning,
|
||||
"BinaryRecorder: free disk below MinDiskFreeMB; recording stopped");
|
||||
}
|
||||
armed_ = false;
|
||||
backLen_ = 0u;
|
||||
CloseFile();
|
||||
return;
|
||||
}
|
||||
diskLow_ = false;
|
||||
|
||||
/* Open a file if none is active. */
|
||||
if ((fd_ < 0) && (pushHeaderLen_ > 0u)) {
|
||||
OpenNewFile();
|
||||
}
|
||||
if (fd_ < 0) {
|
||||
backLen_ = 0u;
|
||||
return;
|
||||
}
|
||||
|
||||
/* Flush staged rows. */
|
||||
if (backLen_ > 0u) {
|
||||
const ssize_t w = pwrite(fd_, back_, backLen_,
|
||||
static_cast<off_t>(fileOffset_));
|
||||
if (w > 0) {
|
||||
fileOffset_ += static_cast<uint64>(w);
|
||||
bytesWritten_ += static_cast<uint64>(w);
|
||||
if (pushRowBytes_ > 0u) {
|
||||
rowsWritten_ += static_cast<uint64>(w) / pushRowBytes_;
|
||||
}
|
||||
}
|
||||
backLen_ = 0u;
|
||||
}
|
||||
|
||||
/* Rotate on size cap. */
|
||||
if (fileOffset_ >= static_cast<uint64>(config_.maxFileBytes)) {
|
||||
CloseFile();
|
||||
OpenNewFile();
|
||||
PruneOldFiles();
|
||||
}
|
||||
|
||||
/* Periodic durability. */
|
||||
if ((fd_ >= 0) &&
|
||||
((nowSec - lastSyncSec_) >= config_.flushIntervalSec)) {
|
||||
(void) fdatasync(fd_);
|
||||
lastSyncSec_ = nowSec;
|
||||
}
|
||||
}
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
/* File management (push thread) */
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
void BinaryRecorder::OpenNewFile() {
|
||||
struct timeval tv;
|
||||
(void) gettimeofday(&tv, static_cast<struct timezone *>(0));
|
||||
struct tm tmv;
|
||||
time_t secs = static_cast<time_t>(tv.tv_sec);
|
||||
(void) gmtime_r(&secs, &tmv);
|
||||
char stamp[32];
|
||||
(void) strftime(stamp, sizeof(stamp), "%Y%m%dT%H%M%S", &tmv);
|
||||
|
||||
fileSeq_++;
|
||||
(void) snprintf(curFilePath_, sizeof(curFilePath_),
|
||||
"%s/%s_%s_%08u.bin",
|
||||
config_.directory, sourceId_, stamp, fileSeq_);
|
||||
|
||||
fd_ = open(curFilePath_, O_WRONLY | O_CREAT | O_TRUNC, 0664);
|
||||
if (fd_ < 0) {
|
||||
REPORT_ERROR_STATIC(MARTe::ErrorManagement::Warning,
|
||||
"BinaryRecorder: failed to open recording file");
|
||||
fileOffset_ = 0u;
|
||||
return;
|
||||
}
|
||||
(void) pwrite(fd_, pushHeader_, pushHeaderLen_, 0);
|
||||
fileOffset_ = pushHeaderLen_;
|
||||
}
|
||||
|
||||
void BinaryRecorder::CloseFile() {
|
||||
if (fd_ >= 0) {
|
||||
(void) fdatasync(fd_);
|
||||
(void) close(fd_);
|
||||
fd_ = -1;
|
||||
}
|
||||
fileOffset_ = 0u;
|
||||
}
|
||||
|
||||
void BinaryRecorder::PruneOldFiles() {
|
||||
DIR *dir = opendir(config_.directory);
|
||||
if (dir == static_cast<DIR *>(0)) { return; }
|
||||
|
||||
char prefix[160];
|
||||
(void) snprintf(prefix, sizeof(prefix), "%s_", sourceId_);
|
||||
const size_t prefixLen = strlen(prefix);
|
||||
|
||||
static const uint32 kMaxList = 4096u;
|
||||
char (*names)[256] = new char[kMaxList][256];
|
||||
uint32 count = 0u;
|
||||
|
||||
struct dirent *ent;
|
||||
while (((ent = readdir(dir)) != static_cast<struct dirent *>(0)) &&
|
||||
(count < kMaxList)) {
|
||||
const char *nm = ent->d_name;
|
||||
const size_t nlen = strlen(nm);
|
||||
if (nlen < prefixLen + 4u) { continue; }
|
||||
if (strncmp(nm, prefix, prefixLen) != 0) { continue; }
|
||||
if (strcmp(nm + (nlen - 4u), ".bin") != 0) { continue; }
|
||||
strncpy(names[count], nm, 255u);
|
||||
names[count][255] = '\0';
|
||||
count++;
|
||||
}
|
||||
(void) closedir(dir);
|
||||
|
||||
/* Sort ascending (lexicographic == chronological via timestamp+seq). */
|
||||
for (uint32 a = 0u; a < count; a++) {
|
||||
for (uint32 b = a + 1u; b < count; b++) {
|
||||
if (strcmp(names[a], names[b]) > 0) {
|
||||
char t[256];
|
||||
strncpy(t, names[a], 256u);
|
||||
strncpy(names[a], names[b], 256u);
|
||||
strncpy(names[b], t, 256u);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Unlink oldest until at most keepFiles remain. */
|
||||
if (count > config_.keepFiles) {
|
||||
const uint32 toRemove = count - config_.keepFiles;
|
||||
for (uint32 i = 0u; i < toRemove; i++) {
|
||||
char full[768];
|
||||
(void) snprintf(full, sizeof(full), "%s/%s",
|
||||
config_.directory, names[i]);
|
||||
(void) unlink(full);
|
||||
}
|
||||
}
|
||||
|
||||
delete[] names;
|
||||
}
|
||||
|
||||
bool BinaryRecorder::HasSufficientDisk() {
|
||||
if (config_.minDiskFreeMB == 0u) { return true; }
|
||||
struct statvfs st;
|
||||
if (statvfs(config_.directory, &st) != 0) { return true; }
|
||||
const uint64 freeBytes = static_cast<uint64>(st.f_bavail) *
|
||||
static_cast<uint64>(st.f_frsize);
|
||||
const uint64 freeMB = freeBytes / (1024u * 1024u);
|
||||
return freeMB >= static_cast<uint64>(config_.minDiskFreeMB);
|
||||
}
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
/* Control + info */
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
void BinaryRecorder::RequestArm() {
|
||||
(void) stagingMutex_.FastLock();
|
||||
pendingArm_ = true;
|
||||
pendingDisarm_ = false;
|
||||
stagingMutex_.FastUnLock();
|
||||
}
|
||||
|
||||
void BinaryRecorder::RequestDisarm() {
|
||||
(void) stagingMutex_.FastLock();
|
||||
pendingDisarm_ = true;
|
||||
pendingArm_ = false;
|
||||
stagingMutex_.FastUnLock();
|
||||
}
|
||||
|
||||
bool BinaryRecorder::IsEnabled() const {
|
||||
return config_.enabled;
|
||||
}
|
||||
|
||||
void BinaryRecorder::GetInfo(bool &recording, char *file, uint32 fileSz,
|
||||
uint64 &bytesWritten, uint64 &rowsWritten,
|
||||
uint64 &droppedRows, uint64 &freeMB) const {
|
||||
recording = armed_ && (fd_ >= 0);
|
||||
if ((file != static_cast<char *>(0)) && (fileSz > 0u)) {
|
||||
strncpy(file, curFilePath_, fileSz - 1u);
|
||||
file[fileSz - 1u] = '\0';
|
||||
}
|
||||
bytesWritten = bytesWritten_;
|
||||
rowsWritten = rowsWritten_;
|
||||
droppedRows = droppedRows_;
|
||||
|
||||
freeMB = 0u;
|
||||
struct statvfs st;
|
||||
if (statvfs(config_.directory, &st) == 0) {
|
||||
const uint64 freeBytes = static_cast<uint64>(st.f_bavail) *
|
||||
static_cast<uint64>(st.f_frsize);
|
||||
freeMB = freeBytes / (1024u * 1024u);
|
||||
}
|
||||
}
|
||||
|
||||
} /* namespace StreamHub */
|
||||
@@ -0,0 +1,164 @@
|
||||
/**
|
||||
* @file BinaryRecorder.h
|
||||
* @brief Per-source binary recorder writing MARTe2 FileWriter-compatible files.
|
||||
*
|
||||
* @copyright Copyright 2015 F4E | European Joint Undertaking for ITER and
|
||||
* the Development of Fusion Energy ('Fusion for Energy').
|
||||
* Licensed under the EUPL, Version 1.1 or - as soon they will be approved
|
||||
* by the European Commission - subsequent versions of the EUPL (the "Licence")
|
||||
* You may not use this work except in compliance with the Licence.
|
||||
* You may obtain a copy of the Licence at: http://ec.europa.eu/idabc/eupl
|
||||
*
|
||||
* @warning Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the Licence is distributed on an "AS IS"
|
||||
* basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
* or implied. See the Licence permissions and limitations under the Licence.
|
||||
*
|
||||
* @details
|
||||
* Records incoming signal packets to disk in the MARTe2 FileWriter binary
|
||||
* format, one rotating file per source. The capture path (CapturePacket,
|
||||
* Configure) runs on the session receive thread; all file I/O (FlushTick)
|
||||
* runs on the 30 Hz push thread. Only the staging double-buffer, the header
|
||||
* blob and a handful of control flags cross threads, guarded by a
|
||||
* FastPollingMutexSem.
|
||||
*
|
||||
* On-disk format (byte-identical to FileWriter for un-quantized signals):
|
||||
* [uint32 numSigs]
|
||||
* per signal: [uint16 TypeDescriptor.all][char[32] name][uint32 numElements]
|
||||
* rows: each row = all included signals concatenated, native little-endian.
|
||||
*/
|
||||
#ifndef STREAMHUB_BINARYRECORDER_H_
|
||||
#define STREAMHUB_BINARYRECORDER_H_
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "CompilerTypes.h"
|
||||
#include "TypeDescriptor.h"
|
||||
#include "FastPollingMutexSem.h"
|
||||
#include "UDPSProtocol.h"
|
||||
|
||||
namespace StreamHub {
|
||||
|
||||
using MARTe::uint8;
|
||||
using MARTe::uint16;
|
||||
using MARTe::uint32;
|
||||
using MARTe::uint64;
|
||||
using MARTe::float64;
|
||||
|
||||
/** Maximum number of signals a recorder can include. */
|
||||
static const uint32 kRecMaxSig = 256u;
|
||||
|
||||
/** Configuration for a BinaryRecorder, parsed from the +Recorder cfg block. */
|
||||
struct RecorderConfig {
|
||||
bool enabled; ///< Master enable
|
||||
bool autoStart; ///< Arm at startup
|
||||
char directory[512]; ///< Output directory
|
||||
uint32 maxFileBytes; ///< Roll threshold (MaxFileMB * 1024 * 1024)
|
||||
uint32 keepFiles; ///< Newest N kept per source
|
||||
uint32 stagingBytes; ///< Per-buffer hard cap (StagingMB * 1024 * 1024)
|
||||
uint32 flushIntervalSec; ///< fdatasync cadence
|
||||
uint32 minDiskFreeMB; ///< Stop-writing guard
|
||||
char signals[1024]; ///< "all" or comma-separated "src:sig" keys
|
||||
};
|
||||
|
||||
class BinaryRecorder {
|
||||
public:
|
||||
BinaryRecorder();
|
||||
~BinaryRecorder();
|
||||
|
||||
/** One-time setup: copy config, source id, create directory. */
|
||||
void Init(const RecorderConfig &cfg, const char *sourceId);
|
||||
|
||||
/**
|
||||
* Receive thread: (re)build the FileWriter header and per-signal row
|
||||
* layout for the included signals. Triggers a file reopen on the push
|
||||
* thread so the new layout takes effect with a fresh header.
|
||||
*/
|
||||
void Configure(const MARTe::UDPSSignalDescriptor *descs, uint32 nSigs,
|
||||
const bool *includeMask);
|
||||
|
||||
/** Receive thread: append one packet's rows to the staging buffer. */
|
||||
void CapturePacket(const uint8 *payload, const uint32 *sigOff,
|
||||
const uint32 *sigElems, uint8 publishMode,
|
||||
uint32 numSamples);
|
||||
|
||||
/** Push thread: open pending file, swap+flush staging, rotate, fsync. */
|
||||
void FlushTick(uint32 nowSec);
|
||||
|
||||
void RequestArm(); ///< Any thread: arm recording.
|
||||
void RequestDisarm(); ///< Any thread: disarm, flush, close.
|
||||
bool IsEnabled() const; ///< True when the master enable is set.
|
||||
|
||||
/** recInfo snapshot (push thread). */
|
||||
void GetInfo(bool &recording, char *file, uint32 fileSz,
|
||||
uint64 &bytesWritten, uint64 &rowsWritten,
|
||||
uint64 &droppedRows, uint64 &freeMB) const;
|
||||
|
||||
/** Map a UDPS type code to a MARTe2 TypeDescriptor.all (uint16). */
|
||||
static uint16 TypeCodeToDescriptorAll(uint8 udpsTypeCode);
|
||||
|
||||
/** Encode a physical value into native little-endian bytes; returns size. */
|
||||
static uint32 EncodeNative(uint8 typeCode, float64 v, uint8 *dst);
|
||||
|
||||
private:
|
||||
void OpenNewFile();
|
||||
void CloseFile();
|
||||
void PruneOldFiles();
|
||||
bool HasSufficientDisk();
|
||||
|
||||
/* Configuration */
|
||||
RecorderConfig config_;
|
||||
char sourceId_[128];
|
||||
|
||||
/* Layout (built on receive thread by Configure) */
|
||||
uint32 srcIdx_[kRecMaxSig]; ///< Source signal index of included signal
|
||||
uint8 typeCode_[kRecMaxSig]; ///< UDPS type code
|
||||
uint8 quantType_[kRecMaxSig]; ///< UDPS quant code
|
||||
uint32 declaredElems_[kRecMaxSig];///< Element count per signal
|
||||
uint32 nativeBytes_[kRecMaxSig]; ///< Native bytes per element
|
||||
uint32 wireBytes_[kRecMaxSig]; ///< Wire bytes per element
|
||||
float64 rangeMin_[kRecMaxSig];
|
||||
float64 rangeMax_[kRecMaxSig];
|
||||
uint32 nIncluded_;
|
||||
uint32 rowBytes_;
|
||||
|
||||
/* Header blob (receive thread writes, push thread snapshots) */
|
||||
uint8 headerBlob_[4u + kRecMaxSig * 38u];
|
||||
uint32 headerLen_;
|
||||
bool layoutReady_;
|
||||
|
||||
/* Push-owned header snapshot (used for file I/O without holding the lock) */
|
||||
uint8 pushHeader_[4u + kRecMaxSig * 38u];
|
||||
uint32 pushHeaderLen_;
|
||||
uint32 pushRowBytes_;
|
||||
|
||||
/* Staging double buffer */
|
||||
uint8 *front_;
|
||||
uint8 *back_;
|
||||
uint32 frontLen_;
|
||||
uint32 backLen_;
|
||||
mutable MARTe::FastPollingMutexSem stagingMutex_;
|
||||
|
||||
/* Control flags (guarded by stagingMutex_) */
|
||||
bool pendingArm_;
|
||||
bool pendingDisarm_;
|
||||
bool pendingReopen_;
|
||||
|
||||
/* Push-thread file state */
|
||||
int fd_;
|
||||
uint64 fileOffset_;
|
||||
uint32 lastSyncSec_;
|
||||
uint32 fileSeq_;
|
||||
char curFilePath_[768];
|
||||
|
||||
/* State + counters */
|
||||
bool armed_;
|
||||
bool diskLow_;
|
||||
uint64 bytesWritten_;
|
||||
uint64 rowsWritten_;
|
||||
uint64 droppedRows_;
|
||||
};
|
||||
|
||||
} /* namespace StreamHub */
|
||||
|
||||
#endif /* STREAMHUB_BINARYRECORDER_H_ */
|
||||
Reference in New Issue
Block a user