Martino Ferrari
908286ee89
A third format beside `.npy` and `.csv`, written by hand in `src/export.cpp` -- MF4 is HDF5 and would mean a library, MF3 is a handful of fixed blocks. The layout is the asammdf 8.8.27 reader's own field list, byte-diffed against a known-good 3.30 file it wrote; the shape is the one ScryLab's data-source contract asks for: one channel group, record-interleaved, the master as the x axis, and the unit in the channel's conversion block. So in MDF the master is `abs_seconds`, named `time` -- the clock that is the same in every display mode -- and the plotted axis goes out as an ordinary channel, `time_relative`. `ExportTable::Column` gained the CONFIG's unit, which only this writer spells out; the `.npy`/`.csv` bytes are unchanged, and `write_export()` now picks the writer for both the dialog and the probe.
648 lines
29 KiB
C++
648 lines
29 KiB
C++
// SPDX-License-Identifier: MIT
|
|
// SPDX-FileCopyrightText: 2026 Martino Ferrari
|
|
|
|
#include "export.hpp"
|
|
|
|
#include <algorithm>
|
|
#include <bit>
|
|
#include <cerrno>
|
|
#include <cstdint>
|
|
#include <cstdio>
|
|
#include <cstring>
|
|
#include <ctime>
|
|
#include <utility>
|
|
|
|
namespace udpscope {
|
|
namespace {
|
|
|
|
// The writers memcpy `f64`/`f32` straight out and label them `<f8`/`<f4`, so a
|
|
// big-endian host would produce a file whose bytes contradict its own dtype. There is no
|
|
// such host in this app's reach and a byte-swapping path nobody runs is a path nobody
|
|
// tests, so the assumption is stated instead of handled.
|
|
static_assert(std::endian::native == std::endian::little,
|
|
"the .npy writer labels its fields little-endian and does not swap");
|
|
|
|
/// A column label a reader can use: NumPy rejects duplicate field names outright, and a
|
|
/// quote or a brace in one would break the header dict it is spelled into. Signal names
|
|
/// come from the stream, so neither is this app's to promise.
|
|
std::string sanitise(const std::string& name) {
|
|
std::string out;
|
|
out.reserve(name.size());
|
|
for (const char ch : name) {
|
|
const bool ok = (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||
|
|
(ch >= '0' && ch <= '9') || ch == '_';
|
|
out.push_back(ok ? ch : '_');
|
|
}
|
|
if (out.empty()) out = "col";
|
|
return out;
|
|
}
|
|
|
|
/// `base` carrying a `_2`, `_3` … suffix for the `n`-th copy (`n == 1` is the name itself),
|
|
/// cut to `max_len` bytes when one is given -- MDF's name field is 32 bytes wide and the file
|
|
/// is re-read by a reader that keeps only the *first* occurrence of a name, so a cut that made
|
|
/// two channels equal would silently lose one.
|
|
std::string numbered(const std::string& base, int n, std::size_t max_len) {
|
|
const std::string suffix = n == 1 ? std::string() : "_" + std::to_string(n);
|
|
std::size_t room = base.size();
|
|
if (max_len != 0) room = max_len > suffix.size() ? max_len - suffix.size() : 0;
|
|
return base.substr(0, room) + suffix;
|
|
}
|
|
|
|
/// The final labels: sanitised, and de-duplicated against each other *and* against
|
|
/// `taken` -- the names the writer already spends -- with a `_2`, `_3` suffix. Two
|
|
/// elements of one vector signal can easily sanitise to the same string.
|
|
std::vector<std::string> column_labels(const ExportTable& table, std::vector<std::string> taken,
|
|
std::size_t max_len = 0) {
|
|
std::vector<std::string> out;
|
|
out.reserve(table.columns.size());
|
|
for (const ExportTable::Column& col : table.columns) {
|
|
const std::string base = sanitise(col.name);
|
|
std::string label = numbered(base, 1, max_len);
|
|
for (int n = 2; std::find(taken.begin(), taken.end(), label) != taken.end(); ++n) {
|
|
label = numbered(base, n, max_len);
|
|
}
|
|
taken.push_back(label);
|
|
out.push_back(std::move(label));
|
|
}
|
|
return out;
|
|
}
|
|
|
|
struct File {
|
|
explicit File(const std::string& path) : f(std::fopen(path.c_str(), "wb")) {}
|
|
~File() {
|
|
if (f != nullptr) std::fclose(f);
|
|
}
|
|
File(const File&) = delete;
|
|
File& operator=(const File&) = delete;
|
|
std::FILE* f = nullptr;
|
|
};
|
|
|
|
bool opened(const File& out, const std::string& path, std::string& error) {
|
|
if (out.f != nullptr) return true;
|
|
error = "cannot write '" + path + "': " + std::strerror(errno);
|
|
return false;
|
|
}
|
|
|
|
/// The block lengths of MDF 3.30, from the **reader's** own field lists
|
|
/// (`asammdf/blocks/v2_v3_constants.py`, 8.8.27): every block is a two-letter id, a 16-bit
|
|
/// length and then its own fields, and the reader walks the file by those lengths alone, so a
|
|
/// block written one field too wide is a file whose reader lands in the middle of the next one.
|
|
/// The widths here are the ones a `bytes(block)` of the reference file measures (the known-good
|
|
/// 3.30 file `/tmp/ref.mdf` this writer was byte-diffed against).
|
|
constexpr std::size_t kMdfIdBlock = 64; // "MDF 3.30" and the flags, no id/length
|
|
constexpr std::size_t kMdfHeader = 208; // 164 common + 44 of the 3.2-era time fields
|
|
constexpr std::size_t kMdfTxOverhead = 5; // "TX", the u16 length and the terminating NUL
|
|
constexpr std::size_t kMdfDataGroup = 28; // 4 addresses, the group count, 4 reserved
|
|
constexpr std::size_t kMdfConversion = 46; // a NONE conversion is the common fields alone
|
|
constexpr std::size_t kMdfChannel = 228; // the display-name-length CNBLOCK
|
|
constexpr std::size_t kMdfChannelGroup = 30; // 26 + the sample-reduction address of 3.30
|
|
constexpr std::size_t kMdfNameBytes = 31; // a CNBLOCK's 32-byte name field, less its NUL
|
|
|
|
/// A byte sink for the MDF header blocks, which are a few hundred bytes each and have to be
|
|
/// laid out field by field. The little-endian `raw()` writes are the same assumption the `.npy`
|
|
/// writer states at the top of this file, and `static_assert`s there.
|
|
struct Bytes {
|
|
std::vector<unsigned char> v;
|
|
|
|
void raw(const void* p, std::size_t n) {
|
|
const auto* b = static_cast<const unsigned char*>(p);
|
|
v.insert(v.end(), b, b + n);
|
|
}
|
|
void u16(std::uint16_t x) {
|
|
const unsigned char b[2] = {static_cast<unsigned char>(x & 0xFFu),
|
|
static_cast<unsigned char>((x >> 8) & 0xFFu)};
|
|
raw(b, sizeof(b));
|
|
}
|
|
void i16(std::int16_t x) { u16(static_cast<std::uint16_t>(x)); }
|
|
void u32(std::uint32_t x) {
|
|
const unsigned char b[4] = {static_cast<unsigned char>(x & 0xFFu),
|
|
static_cast<unsigned char>((x >> 8) & 0xFFu),
|
|
static_cast<unsigned char>((x >> 16) & 0xFFu),
|
|
static_cast<unsigned char>((x >> 24) & 0xFFu)};
|
|
raw(b, sizeof(b));
|
|
}
|
|
void u64(std::uint64_t x) {
|
|
u32(static_cast<std::uint32_t>(x & 0xFFFFFFFFu));
|
|
u32(static_cast<std::uint32_t>(x >> 32));
|
|
}
|
|
void f64(double x) { raw(&x, sizeof(x)); }
|
|
void f32(float x) { raw(&x, sizeof(x)); }
|
|
void zeros(std::size_t n) { v.insert(v.end(), n, 0); }
|
|
|
|
/// A fixed-width text field: the bytes, then NUL padding. MDF's names and units are
|
|
/// NUL-terminated inside a field of a width the reader is told by the block size, so this is
|
|
/// how every string in the format is spelled.
|
|
void text(const std::string& s, std::size_t width) {
|
|
const std::size_t n = std::min(s.size(), width);
|
|
raw(s.data(), n);
|
|
zeros(width - n);
|
|
}
|
|
|
|
std::size_t size() const noexcept { return v.size(); }
|
|
};
|
|
|
|
/// Checked rather than trusted: a field written at the wrong width would not fail here, it
|
|
/// would produce a file whose reader walks into the middle of the next block. The widths are
|
|
/// the asammdf 8.8.27 reader's own field lists (`blocks/v2_v3_constants.py`).
|
|
bool mdf_block_size(const Bytes& block, std::size_t want, const char* what, std::string& error) {
|
|
if (block.size() == want) return true;
|
|
error = std::string("internal: the MDF ") + what + " block is " + std::to_string(block.size()) +
|
|
" bytes, the layout wants " + std::to_string(want);
|
|
return false;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
ExportTable collect_export(ScopeModel& model, RowSet& rows) {
|
|
ExportTable table;
|
|
const RowSet::Row& r0 = rows.row(0);
|
|
if (r0.plot == nullptr) return table;
|
|
const plot::Range view = r0.plot->x_range(r0.axis_x);
|
|
|
|
// Which channels: every bound slot on a visible row, de-duplicated, then sorted. A
|
|
// signal dropped on two rows is one column, and channel order is the signal list's
|
|
// order, so the columns read in the same order as the panel.
|
|
std::vector<int> chans;
|
|
for (int i = 0; i < rows.row_count(); ++i) {
|
|
for (int s = 0; s < RowSet::kSlotsPerRow; ++s) {
|
|
const int c = rows.channel_of(i, s);
|
|
if (c >= 0 && std::find(chans.begin(), chans.end(), c) == chans.end()) {
|
|
chans.push_back(c);
|
|
}
|
|
}
|
|
}
|
|
std::sort(chans.begin(), chans.end());
|
|
if (chans.empty()) return table;
|
|
|
|
const std::vector<ChannelInfo>& infos = model.channels();
|
|
const auto name_of = [&](int c) {
|
|
return static_cast<std::size_t>(c) < infos.size() ? infos[static_cast<std::size_t>(c)].name
|
|
: std::string("ch") + std::to_string(c);
|
|
};
|
|
// The unit travels with the column so the MDF writer can put it on the channel; the other
|
|
// writers ignore it, as they always have.
|
|
const auto unit_of = [&](int c) {
|
|
return static_cast<std::size_t>(c) < infos.size() ? infos[static_cast<std::size_t>(c)].unit
|
|
: std::string();
|
|
};
|
|
|
|
// What the rows bind to is what gets exported, so the split is the same `DisplayMode`
|
|
// `RowSet::bind()` switches on: the frozen event only while one is held, the live ring
|
|
// otherwise -- including mid-capture, where the rows follow the live ring too. Reading
|
|
// the ring while an event is on screen would export data the user cannot see, and
|
|
// reading the stale snapshot mid-capture would export the *previous* event.
|
|
// What is on screen: a pause's frozen view, or the trigger's held event. Both are immutable
|
|
// snapshots, so the export is the picture the user is looking at either way.
|
|
std::shared_ptr<const EventSnapshot> snap = model.display_snapshot();
|
|
// The absolute clock of display x = 0, so `abs_base + time[i]` is the number the stream
|
|
// stamped sample i with -- **one expression, correct in all four display modes**, which is the
|
|
// whole reason `stream_at_x0()` exists. `epoch()` is the stream time of the first sample ever
|
|
// seen (the rings store stamps relative to it) and `stream_at_x0()` is the offset of the
|
|
// display's origin within that, whether it comes from a snapshot or from the live epoch. The
|
|
// snapshot branch used to add `trigger_time()` here, which named an unrelated edge on a
|
|
// *paused* export, or none at all before the first latch.
|
|
const f64 abs_base = model.epoch() + model.stream_at_x0();
|
|
if (snap != nullptr) {
|
|
// The snapshot is immutable and already index-aligned, so the visible window is two
|
|
// binary searches over its stamps and the same index range for every channel.
|
|
const std::vector<f64>& st = snap->stamps;
|
|
const auto lo = std::lower_bound(st.begin(), st.end(), view.min);
|
|
const auto hi = std::upper_bound(st.begin(), st.end(), view.max);
|
|
if (hi <= lo) return table;
|
|
const std::size_t first = static_cast<std::size_t>(lo - st.begin());
|
|
const std::size_t n = static_cast<std::size_t>(hi - lo);
|
|
table.time.resize(n);
|
|
table.abs_seconds.resize(n);
|
|
for (std::size_t i = 0; i < n; ++i) {
|
|
table.time[i] = st[first + i];
|
|
table.abs_seconds[i] = snap->abs_stamps.empty()
|
|
? abs_base + table.time[i]
|
|
: snap->abs_stamps[first + i];
|
|
}
|
|
for (const int c : chans) {
|
|
const std::size_t ch = static_cast<std::size_t>(c);
|
|
if (ch >= snap->values.size()) continue;
|
|
const std::vector<f32>& src = snap->values[ch];
|
|
if (src.size() < first + n) continue;
|
|
table.columns.push_back({name_of(c), unit_of(c),
|
|
std::vector<f32>(src.begin() + static_cast<std::ptrdiff_t>(first),
|
|
src.begin() + static_cast<std::ptrdiff_t>(first + n))});
|
|
}
|
|
return table;
|
|
}
|
|
|
|
// Live: one absolute index range over the shared time ring, used for every channel --
|
|
// `live_window_locked()` is the same cut a latch makes, so an export and a capture of
|
|
// the same window hold the same samples. The lock is against a structural change
|
|
// dropping the rings under the copy, which is what the UI thread has to fear here.
|
|
std::lock_guard<std::mutex> lock(model.mutex());
|
|
// The axis reads `stamp - stream_at_x0()`, so the window has to be put back on the stream's
|
|
// clock before the ring is searched -- and that offset is a whole window while streaming now,
|
|
// not the 0 it once was: the live axis is `[-window, 0]` about the newest sample.
|
|
const f64 s0 = model.stream_at_x0();
|
|
u64 first = 0, last = 0;
|
|
model.live_window_locked(view.min + s0, view.max + s0, first, last);
|
|
const u64 n = last - first;
|
|
if (n == 0) return table;
|
|
|
|
BufferStore& store = model.live();
|
|
std::vector<f64> stamps(static_cast<std::size_t>(n));
|
|
store.time().copy_from(first, n, stamps.data());
|
|
table.time.resize(static_cast<std::size_t>(n));
|
|
table.abs_seconds.resize(static_cast<std::size_t>(n));
|
|
std::vector<f64> abs_stamps;
|
|
if (const plot::data::RingBuffer<f64>* abs_ring = store.abs_time(); abs_ring != nullptr) {
|
|
abs_stamps.resize(static_cast<std::size_t>(n));
|
|
abs_ring->copy_from(first, n, abs_stamps.data());
|
|
}
|
|
for (std::size_t i = 0; i < stamps.size(); ++i) {
|
|
table.time[i] = stamps[i] - s0;
|
|
table.abs_seconds[i] = abs_stamps.empty()
|
|
? abs_base + table.time[i]
|
|
: model.epoch() + abs_stamps[i];
|
|
}
|
|
for (const int c : chans) {
|
|
const std::size_t ch = static_cast<std::size_t>(c);
|
|
if (ch >= model.channel_count()) continue;
|
|
std::vector<f32> values(static_cast<std::size_t>(n));
|
|
store.values(ch).copy_from(first, n, values.data());
|
|
table.columns.push_back({name_of(c), unit_of(c), std::move(values)});
|
|
}
|
|
return table;
|
|
}
|
|
|
|
ExportTable event_table(ScopeModel& model, const EventSnapshot& snapshot) {
|
|
ExportTable table;
|
|
table.time = snapshot.stamps;
|
|
table.abs_seconds = snapshot.abs_stamps.empty() ? snapshot.stamps : snapshot.abs_stamps;
|
|
std::lock_guard<std::mutex> lock(model.mutex());
|
|
const std::vector<ChannelInfo>& channels = model.channels();
|
|
for (std::size_t c = 0; c < snapshot.values.size() && c < channels.size(); ++c) {
|
|
ExportTable::Column column;
|
|
column.name = channels[c].name;
|
|
column.unit = channels[c].unit;
|
|
column.values = snapshot.values[c];
|
|
table.columns.push_back(std::move(column));
|
|
}
|
|
return table;
|
|
}
|
|
|
|
bool write_npy(const std::string& path, const ExportTable& table, std::string& error) {
|
|
if (table.empty()) {
|
|
error = "nothing to export: no signal is plotted in the visible range";
|
|
return false;
|
|
}
|
|
const std::vector<std::string> labels = column_labels(table, {"time", "abs_seconds"});
|
|
|
|
// `descr` as a list of (name, format) pairs is a *packed* structured dtype -- NumPy
|
|
// only inserts alignment padding when asked to (`align=True`) -- so the records below
|
|
// are field widths back to back with no gaps.
|
|
std::string descr = "[('time', '<f8'), ('abs_seconds', '<f8')";
|
|
for (const std::string& label : labels) descr += ", ('" + label + "', '<f4')";
|
|
descr += "]";
|
|
std::string header = "{'descr': " + descr + ", 'fortran_order': False, 'shape': (" +
|
|
std::to_string(table.rows()) + ",), }";
|
|
// v1.0: a 6-byte magic, 2 version bytes and a 2-byte little-endian header length, and
|
|
// the whole preamble padded so the data starts on a 64-byte boundary.
|
|
constexpr std::size_t kPreamble = 10;
|
|
const std::size_t unpadded = kPreamble + header.size() + 1;
|
|
const std::size_t padded = (unpadded + 63) / 64 * 64;
|
|
header.append(padded - unpadded, ' ');
|
|
header.push_back('\n');
|
|
if (header.size() > 0xFFFFu) {
|
|
error = "too many columns for a v1.0 .npy header";
|
|
return false;
|
|
}
|
|
|
|
File out(path);
|
|
if (!opened(out, path, error)) return false;
|
|
const unsigned char len[2] = {static_cast<unsigned char>(header.size() & 0xFFu),
|
|
static_cast<unsigned char>((header.size() >> 8) & 0xFFu)};
|
|
std::fwrite("\x93NUMPY\x01\x00", 1, 8, out.f);
|
|
std::fwrite(len, 1, 2, out.f);
|
|
std::fwrite(header.data(), 1, header.size(), out.f);
|
|
|
|
// One record at a time, into a reused buffer: a column-major staging array would be a
|
|
// second copy of the whole export, which at a full ring is tens of megabytes.
|
|
std::vector<unsigned char> record(2 * sizeof(f64) + table.columns.size() * sizeof(f32));
|
|
for (std::size_t i = 0; i < table.rows(); ++i) {
|
|
std::size_t at = 0;
|
|
std::memcpy(record.data() + at, &table.time[i], sizeof(f64));
|
|
at += sizeof(f64);
|
|
std::memcpy(record.data() + at, &table.abs_seconds[i], sizeof(f64));
|
|
at += sizeof(f64);
|
|
for (const ExportTable::Column& col : table.columns) {
|
|
const f32 v = i < col.values.size() ? col.values[i] : 0.0f;
|
|
std::memcpy(record.data() + at, &v, sizeof(f32));
|
|
at += sizeof(f32);
|
|
}
|
|
if (std::fwrite(record.data(), 1, record.size(), out.f) != record.size()) {
|
|
error = "short write to '" + path + "': " + std::strerror(errno);
|
|
return false;
|
|
}
|
|
}
|
|
if (std::fflush(out.f) != 0) {
|
|
error = "cannot flush '" + path + "': " + std::strerror(errno);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool write_csv(const std::string& path, const ExportTable& table, std::string& error) {
|
|
if (table.empty()) {
|
|
error = "nothing to export: no signal is plotted in the visible range";
|
|
return false;
|
|
}
|
|
File out(path);
|
|
if (!opened(out, path, error)) return false;
|
|
|
|
const std::vector<std::string> labels = column_labels(table, {"time", "abs_seconds"});
|
|
std::string line = "time,abs_seconds";
|
|
for (const std::string& label : labels) line += "," + label;
|
|
line += "\n";
|
|
std::fwrite(line.data(), 1, line.size(), out.f);
|
|
|
|
// `%.17g` for the f64 times and `%.9g` for the f32 values: the shortest widths that
|
|
// round-trip each type exactly. Anything less silently rewrites the samples, which is
|
|
// the one thing an export may not do.
|
|
char buf[64];
|
|
for (std::size_t i = 0; i < table.rows(); ++i) {
|
|
line.clear();
|
|
std::snprintf(buf, sizeof(buf), "%.17g", table.time[i]);
|
|
line += buf;
|
|
std::snprintf(buf, sizeof(buf), "%.17g", table.abs_seconds[i]);
|
|
line += ",";
|
|
line += buf;
|
|
for (const ExportTable::Column& col : table.columns) {
|
|
std::snprintf(buf, sizeof(buf), "%.9g",
|
|
double(i < col.values.size() ? col.values[i] : 0.0f));
|
|
line += ",";
|
|
line += buf;
|
|
}
|
|
line += "\n";
|
|
if (std::fwrite(line.data(), 1, line.size(), out.f) != line.size()) {
|
|
error = "short write to '" + path + "': " + std::strerror(errno);
|
|
return false;
|
|
}
|
|
}
|
|
if (std::fflush(out.f) != 0) {
|
|
error = "cannot flush '" + path + "': " + std::strerror(errno);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool write_mdf(const std::string& path, const ExportTable& table, std::string& error) {
|
|
if (table.empty()) {
|
|
error = "nothing to export: no signal is plotted in the visible range";
|
|
return false;
|
|
}
|
|
const std::size_t rows = table.rows();
|
|
const std::size_t cols = table.columns.size();
|
|
|
|
// **The master is the absolute clock, named `time`**, and the axis the plot showed goes out as
|
|
// an ordinary channel, `time_relative`. The relative column is 0 at a different sample in each
|
|
// display mode, so a master off it would put a live export and a held event's export on two
|
|
// timelines -- the one thing the second column exists to avoid -- while `abs_seconds` is the
|
|
// number the stream itself stamped the samples with. Both carry the unit `s`.
|
|
const std::vector<std::string> labels =
|
|
column_labels(table, {"time", "time_relative"}, kMdfNameBytes);
|
|
const std::size_t channels = cols + 2;
|
|
const std::size_t record = 2 * sizeof(f64) + cols * sizeof(f32);
|
|
if (rows > 0xFFFFFFFFull) {
|
|
error = "too many samples for an MDF 3 file: the channel group counts cycles in 32 bits";
|
|
return false;
|
|
}
|
|
if (128 + (cols - 1) * 32 > 0xFFFFu) {
|
|
error = "too many signals for an MDF 3 record: a channel's start offset is 16 bits";
|
|
return false;
|
|
}
|
|
|
|
// One channel per column, described once: what the CNBLOCK says about it (where in the record
|
|
// its samples start, how wide they are, whether it is the master) and what the CCBLOCK says
|
|
// (its unit). The range is measured from the samples rather than left at zero, so
|
|
// `range_flag = 1` means what it says.
|
|
struct Channel {
|
|
std::string name;
|
|
std::string unit;
|
|
std::uint16_t bit_offset;
|
|
std::uint16_t bit_count;
|
|
std::uint16_t data_type; // MDF 3: 2 = float, 3 = double, both in the ID block's byte order
|
|
std::uint16_t type; // 0 = value, 1 = master
|
|
f64 lo, hi;
|
|
};
|
|
const auto range_of = [](const auto& values) {
|
|
if (values.empty()) return std::pair<f64, f64>{0.0, 0.0};
|
|
const auto mm = std::minmax_element(values.begin(), values.end());
|
|
return std::pair<f64, f64>{static_cast<f64>(*mm.first), static_cast<f64>(*mm.second)};
|
|
};
|
|
std::vector<Channel> chans;
|
|
chans.reserve(channels);
|
|
const auto abs_range = range_of(table.abs_seconds);
|
|
const auto rel_range = range_of(table.time);
|
|
chans.push_back({"time", "s", 0, 64, 3, 1, abs_range.first, abs_range.second});
|
|
chans.push_back({"time_relative", "s", 64, 64, 3, 0, rel_range.first, rel_range.second});
|
|
for (std::size_t i = 0; i < cols; ++i) {
|
|
const auto range = range_of(table.columns[i].values);
|
|
chans.push_back({labels[i], table.columns[i].unit,
|
|
static_cast<std::uint16_t>(128 + i * 32), 32, 2, 0, range.first, range.second});
|
|
}
|
|
|
|
// Every address in the file, from the fixed block lengths above: the writer therefore knows
|
|
// where it is going before it writes a byte, which is what lets each block carry its links.
|
|
const std::string comment = "udpscope export: the visible window, " + std::to_string(rows) +
|
|
" samples x " + std::to_string(cols) + " signals";
|
|
const std::size_t hd_end = kMdfIdBlock + kMdfHeader;
|
|
const std::size_t comment_addr = hd_end;
|
|
const std::size_t data_addr = comment_addr + kMdfTxOverhead + comment.size();
|
|
const std::size_t dg_addr = data_addr + rows * record;
|
|
const std::size_t ch_addr = dg_addr + kMdfDataGroup;
|
|
const std::size_t cg_addr = ch_addr + channels * (kMdfConversion + kMdfChannel);
|
|
const std::uint64_t total = cg_addr + kMdfChannelGroup;
|
|
if (total > 0xFFFFFFFFull) {
|
|
error = "the export is larger than an MDF 3 file can address";
|
|
return false;
|
|
}
|
|
const auto cc_addr = [&](std::size_t i) {
|
|
return ch_addr + i * (kMdfConversion + kMdfChannel);
|
|
};
|
|
|
|
// The wall clock the measurement file was written at. `abs_time` is the same instant in
|
|
// nanoseconds since the epoch, which is what the 3.2-era header field asks for.
|
|
const std::time_t now = std::time(nullptr);
|
|
std::tm tm{};
|
|
localtime_r(&now, &tm);
|
|
char date[11] = {};
|
|
char clock[9] = {};
|
|
std::strftime(date, sizeof(date), "%d:%m:%Y", &tm);
|
|
std::strftime(clock, sizeof(clock), "%H:%M:%S", &tm);
|
|
|
|
Bytes head;
|
|
// Identification block: the version string, who wrote the file, and the byte order every
|
|
// sample in it is in.
|
|
head.text("MDF ", 8);
|
|
head.text("3.30", 8);
|
|
head.text("udpscope", 8);
|
|
head.u16(0); // byte order: Intel, ahead of every value and the float format below
|
|
head.u16(0); // float format: IEEE
|
|
head.u16(330); // MDF version, digits without the separator
|
|
head.u16(0); // code page: latin-1
|
|
head.zeros(2); // reserved
|
|
head.zeros(26); // reserved
|
|
head.u16(0); // unfinalised standard flags
|
|
head.u16(0); // unfinalised custom flags
|
|
if (!mdf_block_size(head, kMdfIdBlock, "identification", error)) return false;
|
|
|
|
// Header block, which points at the data group and at the comment that follows it.
|
|
head.raw("HD", 2);
|
|
head.u16(static_cast<std::uint16_t>(kMdfHeader));
|
|
head.u32(static_cast<std::uint32_t>(dg_addr)); // first data group
|
|
head.u32(static_cast<std::uint32_t>(comment_addr));
|
|
head.u32(0); // program block: nothing wrote this but us
|
|
head.u16(1); // data groups
|
|
head.text(date, 10);
|
|
head.text(clock, 8);
|
|
head.text("", 32); // author
|
|
head.text("", 32); // department
|
|
head.text("", 32); // project
|
|
head.text("", 32); // subject
|
|
head.u64(static_cast<std::uint64_t>(now) * 1000000000ull);
|
|
head.i16(static_cast<std::int16_t>(tm.tm_gmtoff / 3600));
|
|
head.u16(0); // time quality
|
|
head.text("Local PC Reference Time", 32);
|
|
if (!mdf_block_size(head, hd_end, "header", error)) return false;
|
|
|
|
// The comment, in the one text block the file carries: plain words rather than the XML a
|
|
// writer that also *edits* files keeps its structured properties in.
|
|
head.raw("TX", 2);
|
|
head.u16(static_cast<std::uint16_t>(comment.size() + kMdfTxOverhead));
|
|
head.raw(comment.data(), comment.size());
|
|
head.zeros(1);
|
|
|
|
Bytes tail;
|
|
// Data group: one channel group, whose channels are chained, and the address of the samples.
|
|
tail.raw("DG", 2);
|
|
tail.u16(static_cast<std::uint16_t>(kMdfDataGroup));
|
|
tail.u32(0); // next data group
|
|
tail.u32(static_cast<std::uint32_t>(cg_addr));
|
|
tail.u32(0); // trigger block
|
|
tail.u32(static_cast<std::uint32_t>(data_addr));
|
|
tail.u16(1); // channel groups
|
|
tail.u16(0); // record IDs: the group is sorted
|
|
tail.zeros(4); // reserved
|
|
if (!mdf_block_size(tail, kMdfDataGroup, "data group", error)) return false;
|
|
|
|
for (std::size_t i = 0; i < channels; ++i) {
|
|
const Channel& c = chans[i];
|
|
// Conversion block: the unit lives here and nowhere else in MDF 3, which is where ScryLab
|
|
// reads a signal's unit from. NONE because the samples are already the physical values --
|
|
// the table was collected as `f32`, so a conversion would only be a second copy of it.
|
|
Bytes cc;
|
|
cc.raw("CC", 2);
|
|
cc.u16(static_cast<std::uint16_t>(kMdfConversion));
|
|
cc.u16(1); // range_flag: the range below is measured, so it is valid
|
|
cc.f64(c.lo);
|
|
cc.f64(c.hi);
|
|
cc.text(c.unit, 20);
|
|
cc.u16(0xFFFFu);
|
|
cc.u16(0); // referenced parameters
|
|
if (!mdf_block_size(cc, kMdfConversion, "conversion", error)) return false;
|
|
|
|
// Channel block: the name ScryLab turns into the signal name, where this channel's samples
|
|
// sit in a record, and the link to the next channel.
|
|
Bytes cn;
|
|
cn.raw("CN", 2);
|
|
cn.u16(static_cast<std::uint16_t>(kMdfChannel));
|
|
cn.u32(i + 1 < channels ? static_cast<std::uint32_t>(cc_addr(i + 1) + kMdfConversion) : 0);
|
|
cn.u32(static_cast<std::uint32_t>(cc_addr(i)));
|
|
cn.u32(0); // source: the stream does not say what produced a sample
|
|
cn.u32(0); // dependency block
|
|
cn.u32(0); // comment
|
|
cn.u16(c.type);
|
|
cn.text(c.name, 32);
|
|
cn.zeros(128); // description
|
|
cn.u16(c.bit_offset);
|
|
cn.u16(c.bit_count);
|
|
cn.u16(c.data_type);
|
|
cn.u16(1); // range_flag
|
|
cn.f64(c.lo);
|
|
cn.f64(c.hi);
|
|
cn.f64(0.0); // sampling rate: only a virtual time channel carries one
|
|
cn.u32(0); // long name
|
|
cn.u32(0); // display name
|
|
cn.u16(0); // additional byte offset
|
|
if (!mdf_block_size(cn, kMdfChannel, "channel", error)) return false;
|
|
|
|
tail.v.insert(tail.v.end(), cc.v.begin(), cc.v.end());
|
|
tail.v.insert(tail.v.end(), cn.v.begin(), cn.v.end());
|
|
}
|
|
|
|
// Channel group, last: it names the first channel, the record size and the cycle count from
|
|
// which a reader derives the size of the sample block at `data_addr`.
|
|
tail.raw("CG", 2);
|
|
tail.u16(static_cast<std::uint16_t>(kMdfChannelGroup));
|
|
tail.u32(0); // next channel group
|
|
tail.u32(static_cast<std::uint32_t>(ch_addr + kMdfConversion));
|
|
tail.u32(0); // comment
|
|
tail.u16(1); // record ID
|
|
tail.u16(static_cast<std::uint16_t>(channels));
|
|
tail.u16(static_cast<std::uint16_t>(record));
|
|
tail.u32(static_cast<std::uint32_t>(rows));
|
|
tail.zeros(4); // sample reduction address
|
|
if (!mdf_block_size(tail, kMdfDataGroup + channels * (kMdfConversion + kMdfChannel) +
|
|
kMdfChannelGroup,
|
|
"data group and channels", error)) {
|
|
return false;
|
|
}
|
|
|
|
File out(path);
|
|
if (!opened(out, path, error)) return false;
|
|
const auto put = [&](const Bytes& bytes) {
|
|
return bytes.v.empty() || std::fwrite(bytes.v.data(), 1, bytes.v.size(), out.f) == bytes.v.size();
|
|
};
|
|
if (!put(head)) {
|
|
error = "short write to '" + path + "': " + std::strerror(errno);
|
|
return false;
|
|
}
|
|
// One record at a time, in the channel order the blocks above declare, so a full ring is not
|
|
// held twice -- the same shape the `.npy` writer streams its records in.
|
|
std::vector<unsigned char> rec(record);
|
|
for (std::size_t i = 0; i < rows; ++i) {
|
|
std::size_t at = 0;
|
|
std::memcpy(rec.data() + at, &table.abs_seconds[i], sizeof(f64));
|
|
at += sizeof(f64);
|
|
std::memcpy(rec.data() + at, &table.time[i], sizeof(f64));
|
|
at += sizeof(f64);
|
|
for (const ExportTable::Column& col : table.columns) {
|
|
const f32 v = i < col.values.size() ? col.values[i] : 0.0f;
|
|
std::memcpy(rec.data() + at, &v, sizeof(f32));
|
|
at += sizeof(f32);
|
|
}
|
|
if (std::fwrite(rec.data(), 1, rec.size(), out.f) != rec.size()) {
|
|
error = "short write to '" + path + "': " + std::strerror(errno);
|
|
return false;
|
|
}
|
|
}
|
|
if (!put(tail)) {
|
|
error = "short write to '" + path + "': " + std::strerror(errno);
|
|
return false;
|
|
}
|
|
if (std::fflush(out.f) != 0) {
|
|
error = "cannot flush '" + path + "': " + std::strerror(errno);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool write_export(const std::string& path, const ExportTable& table, std::string& error) {
|
|
const auto ends_with = [&path](const char* ext) {
|
|
const std::size_t n = std::strlen(ext);
|
|
return path.size() >= n && path.compare(path.size() - n, n, ext) == 0;
|
|
};
|
|
if (ends_with(".csv")) return write_csv(path, table, error);
|
|
if (ends_with(".mf3") || ends_with(".mdf")) return write_mdf(path, table, error);
|
|
return write_npy(path, table, error);
|
|
}
|
|
|
|
} // namespace udpscope
|