fix(StreamHub): correct UTF-8 tail repair to run only after truncation
The previous walk-back in SetCalibrationEntry was unconditional, corrupting short valid units ending in multi-byte characters (e.g. Omega, mu, degree). Also failed to drop an orphaned lead byte left after stripping continuation bytes. Restructured to use a 256-byte staging buffer so truncation can be detected, then repair runs only in the truncation branch. Algorithm now matches Go CalConfig.Normalise() exactly: scan back over continuation bytes (up to 3), find the lead byte, derive expected sequence length, cut if incomplete. Covers all cases: orphaned continuation, orphaned lead, cut on lead byte. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
2f9b135c62
commit
93e00d0c21
@@ -755,33 +755,74 @@ bool StreamHub::SetCalibrationEntry(const char *source, const char *signal,
|
||||
if (!JsonIsFinite(scale) || (scale == 0.0)) { return false; }
|
||||
if (!JsonIsFinite(offset)) { return false; }
|
||||
|
||||
/* Trim unit, then truncate to kMaxUnitLen bytes, walking back any
|
||||
* partial UTF-8 rune to keep the stored bytes valid UTF-8 (Go parity). */
|
||||
/* Trim unit, then — if and only if the trimmed string exceeds kMaxUnitLen
|
||||
* bytes — truncate to kMaxUnitLen and repair the tail so the stored bytes
|
||||
* are valid UTF-8. This exactly mirrors Go's CalConfig.Normalise(): the
|
||||
* walk-back runs only inside the truncation branch, so a short valid string
|
||||
* (e.g. "Ω" = CE A9, 2 bytes) is never touched.
|
||||
*
|
||||
* Repair algorithm (matching Go's utf8.DecodeLastRuneInString loop):
|
||||
* Scan backwards over at most 3 continuation bytes (10xxxxxx, (b&0xC0)==0x80)
|
||||
* to locate the lead byte of the last UTF-8 sequence. Derive the expected
|
||||
* sequence length from that lead byte (0xxxxxxx→1, 110xxxxx→2, 1110xxxx→3,
|
||||
* 11110xxx→4). If the bytes present are fewer than expected, cut the string
|
||||
* at the lead byte. This handles an orphaned continuation byte, an orphaned
|
||||
* lead byte, and the case where the cut lands exactly on the lead byte. */
|
||||
char u[kMaxUnitLen + 1u];
|
||||
u[0] = '\0';
|
||||
if (unit != static_cast<const char *>(0)) {
|
||||
strncpy(u, unit, sizeof(u) - 1u);
|
||||
u[sizeof(u) - 1u] = '\0';
|
||||
TrimInPlace(u);
|
||||
/* Truncate to kMaxUnitLen bytes */
|
||||
if (strlen(u) > kMaxUnitLen) {
|
||||
/* Use a temporary over-sized buffer so we can detect when the trimmed
|
||||
* input is actually longer than kMaxUnitLen (strncpy into u[kMaxUnitLen+1]
|
||||
* would silently cap the copy, making the length check always false). */
|
||||
const uint32 kTmpLen = 256u;
|
||||
char tmp[256u];
|
||||
strncpy(tmp, unit, kTmpLen - 1u);
|
||||
tmp[kTmpLen - 1u] = '\0';
|
||||
TrimInPlace(tmp);
|
||||
|
||||
uint32 tlen = static_cast<uint32>(strlen(tmp));
|
||||
if (tlen <= kMaxUnitLen) {
|
||||
/* Short enough: copy verbatim, no repair needed. */
|
||||
strncpy(u, tmp, kMaxUnitLen);
|
||||
u[kMaxUnitLen] = '\0';
|
||||
}
|
||||
/* Walk back any trailing partial UTF-8 rune. A byte b is a
|
||||
* continuation byte (10xxxxxx) iff (b & 0xC0) == 0x80. A truncation
|
||||
* may leave a sequence starter with fewer continuation bytes than it
|
||||
* expects; drop bytes from the end while the last byte is a lone
|
||||
* continuation byte that decodes as an invalid (RuneError, 1) pair.
|
||||
* Concrete: if the last byte is 0x80-0xBF (continuation), remove it,
|
||||
* then repeat — this matches Go's utf8.DecodeLastRuneInString loop. */
|
||||
uint32 ulen = static_cast<uint32>(strlen(u));
|
||||
while (ulen > 0u) {
|
||||
const unsigned char last = static_cast<unsigned char>(u[ulen - 1u]);
|
||||
/* Is it a UTF-8 continuation byte (10xxxxxx)? */
|
||||
if ((last & 0xC0u) == 0x80u) {
|
||||
u[--ulen] = '\0';
|
||||
} else {
|
||||
/* Truncate at kMaxUnitLen bytes, then repair any split rune. */
|
||||
strncpy(u, tmp, kMaxUnitLen);
|
||||
u[kMaxUnitLen] = '\0';
|
||||
uint32 ulen = kMaxUnitLen;
|
||||
|
||||
/* Scan back over continuation bytes (up to 3). */
|
||||
uint32 cont = 0u;
|
||||
while ((cont < 3u) && (cont < ulen)) {
|
||||
const unsigned char b =
|
||||
static_cast<unsigned char>(u[ulen - 1u - cont]);
|
||||
if ((b & 0xC0u) == 0x80u) {
|
||||
cont++;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* The byte at index ulen-1-cont is the candidate lead byte. */
|
||||
if (cont < ulen) {
|
||||
const unsigned char lead =
|
||||
static_cast<unsigned char>(u[ulen - 1u - cont]);
|
||||
uint32 expected = 0u;
|
||||
if ((lead & 0x80u) == 0x00u) { expected = 1u; }
|
||||
else if ((lead & 0xE0u) == 0xC0u) { expected = 2u; }
|
||||
else if ((lead & 0xF0u) == 0xE0u) { expected = 3u; }
|
||||
else if ((lead & 0xF8u) == 0xF0u) { expected = 4u; }
|
||||
/* bytes present in the sequence = cont + 1 (the lead itself) */
|
||||
if ((expected > 1u) && ((cont + 1u) < expected)) {
|
||||
/* Incomplete multi-byte sequence: drop from the lead byte. */
|
||||
ulen = ulen - 1u - cont;
|
||||
u[ulen] = '\0';
|
||||
}
|
||||
/* else: complete sequence (expected==1 ASCII, or cont+1==expected)
|
||||
* — nothing to do. */
|
||||
} else {
|
||||
break;
|
||||
/* Every byte was a continuation byte with no lead: discard all. */
|
||||
u[0] = '\0';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user