fix(StreamHub): correct UTF-8 tail repair to run only after truncation

The previous walk-back in SetCalibrationEntry was unconditional, corrupting
short valid units ending in multi-byte characters (e.g. Omega, mu, degree).
Also failed to drop an orphaned lead byte left after stripping continuation
bytes. Restructured to use a 256-byte staging buffer so truncation can be
detected, then repair runs only in the truncation branch. Algorithm now
matches Go CalConfig.Normalise() exactly: scan back over continuation bytes
(up to 3), find the lead byte, derive expected sequence length, cut if
incomplete. Covers all cases: orphaned continuation, orphaned lead, cut on
lead byte.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Martino Ferrari
2026-08-17 00:02:33 +02:00
co-authored by Claude Sonnet 4.6
parent 2f9b135c62
commit 93e00d0c21
2 changed files with 120 additions and 22 deletions
+63 -22
View File
@@ -755,33 +755,74 @@ bool StreamHub::SetCalibrationEntry(const char *source, const char *signal,
if (!JsonIsFinite(scale) || (scale == 0.0)) { return false; }
if (!JsonIsFinite(offset)) { return false; }
/* Trim unit, then truncate to kMaxUnitLen bytes, walking back any
* partial UTF-8 rune to keep the stored bytes valid UTF-8 (Go parity). */
/* Trim unit, then — if and only if the trimmed string exceeds kMaxUnitLen
* bytes — truncate to kMaxUnitLen and repair the tail so the stored bytes
* are valid UTF-8. This exactly mirrors Go's CalConfig.Normalise(): the
* walk-back runs only inside the truncation branch, so a short valid string
* (e.g. "Ω" = CE A9, 2 bytes) is never touched.
*
* Repair algorithm (matching Go's utf8.DecodeLastRuneInString loop):
* Scan backwards over at most 3 continuation bytes (10xxxxxx, (b&0xC0)==0x80)
* to locate the lead byte of the last UTF-8 sequence. Derive the expected
* sequence length from that lead byte (0xxxxxxx→1, 110xxxxx→2, 1110xxxx→3,
* 11110xxx→4). If the bytes present are fewer than expected, cut the string
* at the lead byte. This handles an orphaned continuation byte, an orphaned
* lead byte, and the case where the cut lands exactly on the lead byte. */
char u[kMaxUnitLen + 1u];
u[0] = '\0';
if (unit != static_cast<const char *>(0)) {
strncpy(u, unit, sizeof(u) - 1u);
u[sizeof(u) - 1u] = '\0';
TrimInPlace(u);
/* Truncate to kMaxUnitLen bytes */
if (strlen(u) > kMaxUnitLen) {
/* Use a temporary over-sized buffer so we can detect when the trimmed
* input is actually longer than kMaxUnitLen (strncpy into u[kMaxUnitLen+1]
* would silently cap the copy, making the length check always false). */
const uint32 kTmpLen = 256u;
char tmp[256u];
strncpy(tmp, unit, kTmpLen - 1u);
tmp[kTmpLen - 1u] = '\0';
TrimInPlace(tmp);
uint32 tlen = static_cast<uint32>(strlen(tmp));
if (tlen <= kMaxUnitLen) {
/* Short enough: copy verbatim, no repair needed. */
strncpy(u, tmp, kMaxUnitLen);
u[kMaxUnitLen] = '\0';
}
/* Walk back any trailing partial UTF-8 rune. A byte b is a
* continuation byte (10xxxxxx) iff (b & 0xC0) == 0x80. A truncation
* may leave a sequence starter with fewer continuation bytes than it
* expects; drop bytes from the end while the last byte is a lone
* continuation byte that decodes as an invalid (RuneError, 1) pair.
* Concrete: if the last byte is 0x80-0xBF (continuation), remove it,
* then repeat — this matches Go's utf8.DecodeLastRuneInString loop. */
uint32 ulen = static_cast<uint32>(strlen(u));
while (ulen > 0u) {
const unsigned char last = static_cast<unsigned char>(u[ulen - 1u]);
/* Is it a UTF-8 continuation byte (10xxxxxx)? */
if ((last & 0xC0u) == 0x80u) {
u[--ulen] = '\0';
} else {
/* Truncate at kMaxUnitLen bytes, then repair any split rune. */
strncpy(u, tmp, kMaxUnitLen);
u[kMaxUnitLen] = '\0';
uint32 ulen = kMaxUnitLen;
/* Scan back over continuation bytes (up to 3). */
uint32 cont = 0u;
while ((cont < 3u) && (cont < ulen)) {
const unsigned char b =
static_cast<unsigned char>(u[ulen - 1u - cont]);
if ((b & 0xC0u) == 0x80u) {
cont++;
} else {
break;
}
}
/* The byte at index ulen-1-cont is the candidate lead byte. */
if (cont < ulen) {
const unsigned char lead =
static_cast<unsigned char>(u[ulen - 1u - cont]);
uint32 expected = 0u;
if ((lead & 0x80u) == 0x00u) { expected = 1u; }
else if ((lead & 0xE0u) == 0xC0u) { expected = 2u; }
else if ((lead & 0xF0u) == 0xE0u) { expected = 3u; }
else if ((lead & 0xF8u) == 0xF0u) { expected = 4u; }
/* bytes present in the sequence = cont + 1 (the lead itself) */
if ((expected > 1u) && ((cont + 1u) < expected)) {
/* Incomplete multi-byte sequence: drop from the lead byte. */
ulen = ulen - 1u - cont;
u[ulen] = '\0';
}
/* else: complete sequence (expected==1 ASCII, or cont+1==expected)
* — nothing to do. */
} else {
break;
/* Every byte was a continuation byte with no lead: discard all. */
u[0] = '\0';
}
}
}