fix: a loaded recording is filtered into its buffer, not truncated
The loader pushed one slot per file row while the plan decimated by eight, so a 10 M-row file overran the ring and only its last 4.19 s survived -- a 20 s recording that read as a 4 s one. Each group of `time_stride` rows is now averaged into one slot at the group's mean stamp, as the packet path already does, and the load line reports rows and decimation so a filtered file cannot be mistaken for a truncated one. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
50a1ddca62
commit
345145c52b
7 files changed
+104
-11
No files matched your search
+8
-4
@@ -76,11 +76,15 @@ int main(int argc, char** argv) {
|
||||
// The whole file framed from the start: the window is the recording's own span, and nothing
|
||||
// afterwards restates it -- a zoom is the user's.
|
||||
if (report.span_s > 0.0) rows.set_time_window(report.span_s);
|
||||
// `rows`/`decim` say whether what is on screen is the file sample for sample or a filtered
|
||||
// version of it: `samples` alone below `rows` is how a recording that did not fit its buffer at
|
||||
// its own rate used to read as a truncated one.
|
||||
std::fprintf(stderr,
|
||||
"[udpscope] waveform loaded: %s signals=%zu samples=%zu span=%.6fs rate=%.6g "
|
||||
"time=[%.6f..%.6f]\n",
|
||||
cfg.open_waveform.c_str(), report.signals, report.samples, report.span_s,
|
||||
report.rate_hz, report.first_time, report.last_time);
|
||||
"[udpscope] waveform loaded: %s signals=%zu samples=%zu rows=%zu decim=%llu "
|
||||
"span=%.6fs rate=%.6g time=[%.6f..%.6f]\n",
|
||||
cfg.open_waveform.c_str(), report.signals, report.samples, report.rows,
|
||||
static_cast<unsigned long long>(report.decimation), report.span_s, report.rate_hz,
|
||||
report.first_time, report.last_time);
|
||||
std::fflush(stderr);
|
||||
}
|
||||
|
||||
|
||||
+46
-4
@@ -8,6 +8,7 @@
|
||||
#include <cmath>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <mutex>
|
||||
#include <sys/stat.h>
|
||||
|
||||
#include "layout_store.hpp"
|
||||
@@ -283,18 +284,57 @@ bool load_waveform_into_model(ScopeModel& model, const std::string& path,
|
||||
if (error != nullptr) *error = "'" + path + "': the model built no channels for its columns";
|
||||
return false;
|
||||
}
|
||||
// The plan the enables above built may **decimate**, and then the file has to be filtered into the
|
||||
// ring exactly as the packet path filters the wire: one stored slot per `time_stride` rows,
|
||||
// carrying their mean. A recording longer than a few seconds at a high rate does not fit a buffer
|
||||
// sized in megabytes -- 20 s of 10 channels at 500 kS/s is 480 MB of rings -- so the planner
|
||||
// decimates, and pushing every row regardless overruns the ring and leaves the file as *its own
|
||||
// tail*: `stored=10000000` against a `published=2097152` ring read as "I loaded a 20 s file and
|
||||
// only the last 4 s are there". Averaging and not striding, for the reason
|
||||
// `invariants-data-path.md` gives: a stride aliases whatever is above the stored Nyquist.
|
||||
std::uint64_t decim = 1;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(model.mutex());
|
||||
if (model.partition().time_stride > 0) decim = model.partition().time_stride;
|
||||
}
|
||||
std::vector<float> values(reader.names().size(), 0.0f);
|
||||
std::vector<float> mean(reader.names().size(), 0.0f);
|
||||
std::vector<double> acc(reader.names().size(), 0.0);
|
||||
std::size_t pushed = 0;
|
||||
std::size_t rows_read = 0;
|
||||
std::uint64_t in_group = 0;
|
||||
double t_acc = 0.0;
|
||||
double abs_acc = 0.0;
|
||||
double stamp = 0.0;
|
||||
double abs_stamp = 0.0;
|
||||
double first = 0.0;
|
||||
double last = 0.0;
|
||||
while (reader.next(&stamp, &abs_stamp, values.data())) {
|
||||
if (pushed == 0) first = stamp;
|
||||
last = stamp;
|
||||
model.push_loaded_slot(values, stamp, abs_stamp);
|
||||
// The group's mean value at its mean stamp, which is where the ingest filter puts it too: a slot
|
||||
// stamped at the group's *first* sample would shift the whole trace half a group early.
|
||||
const auto flush_group = [&]() {
|
||||
const double n = static_cast<double>(in_group);
|
||||
for (std::size_t i = 0; i < acc.size(); ++i) {
|
||||
mean[i] = static_cast<float>(acc[i] / n);
|
||||
acc[i] = 0.0;
|
||||
}
|
||||
model.push_loaded_slot(mean, t_acc / n, abs_acc / n);
|
||||
++pushed;
|
||||
t_acc = 0.0;
|
||||
abs_acc = 0.0;
|
||||
in_group = 0;
|
||||
};
|
||||
while (reader.next(&stamp, &abs_stamp, values.data())) {
|
||||
if (rows_read == 0) first = stamp;
|
||||
last = stamp;
|
||||
++rows_read;
|
||||
for (std::size_t i = 0; i < acc.size(); ++i) acc[i] += values[i];
|
||||
t_acc += stamp;
|
||||
abs_acc += abs_stamp;
|
||||
if (++in_group >= decim) flush_group();
|
||||
}
|
||||
// The last group need not be whole, and dropping it would shorten the record by up to one slot --
|
||||
// which on a one-slot file (`decim` larger than the whole recording) is the entire file.
|
||||
if (in_group > 0) flush_group();
|
||||
reader.close();
|
||||
|
||||
model.set_epoch_if_unset(first);
|
||||
@@ -305,6 +345,8 @@ bool load_waveform_into_model(ScopeModel& model, const std::string& path,
|
||||
if (report != nullptr) {
|
||||
report->signals = channels;
|
||||
report->samples = pushed;
|
||||
report->rows = rows_read;
|
||||
report->decimation = decim;
|
||||
report->span_s = pushed > 0 ? std::abs(last - first) : 0.0;
|
||||
report->rate_hz = rate;
|
||||
report->first_time = first;
|
||||
|
||||
@@ -58,7 +58,14 @@ class WaveformReader {
|
||||
/// What a load put into the model, for the log and the status line.
|
||||
struct WaveformLoadReport {
|
||||
std::size_t signals = 0;
|
||||
/// Slots actually stored, which is the file's row count divided by `decimation`.
|
||||
std::size_t samples = 0;
|
||||
/// Records the file held, and the group size each stored slot is the mean of. They differ when the
|
||||
/// recording does not fit the buffer at its own rate: `samples * decimation == rows` (up to the
|
||||
/// last, partial group), and a `decimation` above 1 is the only honest way to read a `samples`
|
||||
/// below `rows`.
|
||||
std::size_t rows = 0;
|
||||
std::uint64_t decimation = 1;
|
||||
double span_s = 0.0;
|
||||
double rate_hz = 0.0;
|
||||
double first_time = 0.0;
|
||||
|
||||
Reference in new issue
Block a user