StreamHub: loop UTF-8 tail repair to match Go CalConfig.Normalise()
The single-pass repair left invalid bytes when the candidate lead byte had class 0 (illegal 0xF8-0xFF bytes, or a bare continuation byte reached after the 3-byte backward-scan cap). Convert to a loop with a `cut` flag mirroring Go's loop: each iteration either makes no cut (exits) or strictly reduces ulen by >= 1 byte (terminates in <= 16 iterations). Also treat expected==0 as a cut target, matching Go's behaviour of stripping any byte that decodes as an invalid one-byte sequence. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Martino Ferrari
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
93e00d0c21
commit
bdc74f5fd2
@@ -189,3 +189,52 @@ Overall: ALL PASS
|
|||||||
```
|
```
|
||||||
|
|
||||||
All required cases verified: `"Ω"` survives unchanged, `"µs"` survives unchanged, 20-byte ASCII truncates to 16, a cut mid-rune truncates to the last complete rune, and a 16-byte string ending exactly on a complete multi-byte rune is untouched.
|
All required cases verified: `"Ω"` survives unchanged, `"µs"` survives unchanged, 20-byte ASCII truncates to 16, a cut mid-rune truncates to the last complete rune, and a 16-byte string ending exactly on a complete multi-byte rune is untouched.
|
||||||
|
|
||||||
|
## Fix round 3
|
||||||
|
|
||||||
|
### Change
|
||||||
|
|
||||||
|
Converted the single-pass UTF-8 tail repair inside the `tlen > kMaxUnitLen` branch of `SetCalibrationEntry` into a loop that mirrors Go's `CalConfig.Normalise()` exactly. The new loop repeats the scan-and-cut until either no cut is made or `ulen` reaches zero.
|
||||||
|
|
||||||
|
Two new cases are now handled that the old single pass missed:
|
||||||
|
|
||||||
|
1. **Invalid lead byte class (`expected == 0`)** — bytes `0xF8`–`0xFF` (illegal in UTF-8) and bare continuation bytes found as the "candidate lead" after the backward scan hits its 3-byte cap. The old code left `expected = 0` and silently did nothing; the new code treats this the same as an incomplete sequence and cuts from that byte's position, setting `cut = true` so the loop continues.
|
||||||
|
|
||||||
|
2. **Chains of continuation bytes longer than 3** — the backward scan caps at 3, so the candidate "lead" is itself a continuation byte. `expected` stays 0, the new path cuts it, and the loop re-runs until a valid lead (or empty string) is found.
|
||||||
|
|
||||||
|
### Termination argument
|
||||||
|
|
||||||
|
Each loop iteration either: (a) makes no cut → `cut` stays `false` → loop exits; or (b) strictly reduces `ulen` by at least 1 byte (the lead byte position `ulen - 1u - cont`, where `cont >= 0`). Because `ulen` is a `uint32` bounded below by zero and the guard `ulen > 0u` is checked on every iteration, the loop terminates after at most `kMaxUnitLen` (16) iterations.
|
||||||
|
|
||||||
|
### Code diff (StreamHub.cpp, repair block)
|
||||||
|
|
||||||
|
Old: single pass, no loop, `expected == 0` → silent no-op.
|
||||||
|
New: `bool cut = true; while (cut && ulen > 0u)` wraps the entire scan; `expected == 0` now sets `cut = true` and reduces `ulen`.
|
||||||
|
|
||||||
|
### Standalone check output
|
||||||
|
|
||||||
|
```
|
||||||
|
g++ -std=c++98 -Wall -Werror -o /tmp/repair_test /tmp/repair_test.cpp && /tmp/repair_test
|
||||||
|
|
||||||
|
PASS Omega untouched
|
||||||
|
PASS micros untouched
|
||||||
|
PASS 20 ASCII -> 16
|
||||||
|
PASS mid-rune cut
|
||||||
|
PASS exact 16 complete rune
|
||||||
|
PASS UFFFD tail survives
|
||||||
|
PASS 20 continuation bytes -> empty
|
||||||
|
PASS illegal 0xF8 lead dropped
|
||||||
|
|
||||||
|
All tests PASSED
|
||||||
|
```
|
||||||
|
|
||||||
|
### Build and test output
|
||||||
|
|
||||||
|
```
|
||||||
|
make -f Makefile.gcc apps → StreamHub.ex linked successfully (0 errors)
|
||||||
|
|
||||||
|
./Build/x86-linux/GTest/MainGTest.ex
|
||||||
|
132 tests from 12 test cases ran.
|
||||||
|
PASSED: 127
|
||||||
|
FAILED: 5 (UDPStreamerGTest multicast — pre-existing, machine-level issue; expected baseline 127/132 or 128/132)
|
||||||
|
```
|
||||||
|
|||||||
@@ -791,38 +791,55 @@ bool StreamHub::SetCalibrationEntry(const char *source, const char *signal,
|
|||||||
u[kMaxUnitLen] = '\0';
|
u[kMaxUnitLen] = '\0';
|
||||||
uint32 ulen = kMaxUnitLen;
|
uint32 ulen = kMaxUnitLen;
|
||||||
|
|
||||||
/* Scan back over continuation bytes (up to 3). */
|
/* Repair any split or invalid rune at the tail.
|
||||||
uint32 cont = 0u;
|
* Mirror Go's loop: keep stripping until the tail is valid or empty.
|
||||||
while ((cont < 3u) && (cont < ulen)) {
|
* Each iteration either makes no cut (loop exits) or strictly reduces
|
||||||
const unsigned char b =
|
* ulen by at least 1 byte, so termination is guaranteed. */
|
||||||
static_cast<unsigned char>(u[ulen - 1u - cont]);
|
bool cut = true;
|
||||||
if ((b & 0xC0u) == 0x80u) {
|
while (cut && (ulen > 0u)) {
|
||||||
cont++;
|
cut = false;
|
||||||
} else {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* The byte at index ulen-1-cont is the candidate lead byte. */
|
/* Scan back over continuation bytes (up to 3). */
|
||||||
if (cont < ulen) {
|
uint32 cont = 0u;
|
||||||
const unsigned char lead =
|
while ((cont < 3u) && (cont < ulen)) {
|
||||||
static_cast<unsigned char>(u[ulen - 1u - cont]);
|
const unsigned char b =
|
||||||
uint32 expected = 0u;
|
static_cast<unsigned char>(u[ulen - 1u - cont]);
|
||||||
if ((lead & 0x80u) == 0x00u) { expected = 1u; }
|
if ((b & 0xC0u) == 0x80u) {
|
||||||
else if ((lead & 0xE0u) == 0xC0u) { expected = 2u; }
|
cont++;
|
||||||
else if ((lead & 0xF0u) == 0xE0u) { expected = 3u; }
|
} else {
|
||||||
else if ((lead & 0xF8u) == 0xF0u) { expected = 4u; }
|
break;
|
||||||
/* bytes present in the sequence = cont + 1 (the lead itself) */
|
}
|
||||||
if ((expected > 1u) && ((cont + 1u) < expected)) {
|
}
|
||||||
/* Incomplete multi-byte sequence: drop from the lead byte. */
|
|
||||||
ulen = ulen - 1u - cont;
|
/* The byte at index ulen-1-cont is the candidate lead byte. */
|
||||||
u[ulen] = '\0';
|
if (cont < ulen) {
|
||||||
|
const unsigned char lead =
|
||||||
|
static_cast<unsigned char>(u[ulen - 1u - cont]);
|
||||||
|
uint32 expected = 0u;
|
||||||
|
if ((lead & 0x80u) == 0x00u) { expected = 1u; }
|
||||||
|
else if ((lead & 0xE0u) == 0xC0u) { expected = 2u; }
|
||||||
|
else if ((lead & 0xF0u) == 0xE0u) { expected = 3u; }
|
||||||
|
else if ((lead & 0xF8u) == 0xF0u) { expected = 4u; }
|
||||||
|
/* expected==0: lead byte is not a valid UTF-8 lead class
|
||||||
|
* (0xF8-0xFF or a bare continuation); drop it too, like Go. */
|
||||||
|
if (expected == 0u) {
|
||||||
|
/* Invalid lead byte: strip from that position. */
|
||||||
|
ulen = ulen - 1u - cont;
|
||||||
|
u[ulen] = '\0';
|
||||||
|
cut = true;
|
||||||
|
} else if ((cont + 1u) < expected) {
|
||||||
|
/* Incomplete multi-byte sequence: drop from the lead byte. */
|
||||||
|
ulen = ulen - 1u - cont;
|
||||||
|
u[ulen] = '\0';
|
||||||
|
cut = true;
|
||||||
|
}
|
||||||
|
/* else: complete sequence — nothing to do, loop exits. */
|
||||||
|
} else {
|
||||||
|
/* Every byte was a continuation byte with no lead: discard all. */
|
||||||
|
ulen = 0u;
|
||||||
|
u[0] = '\0';
|
||||||
|
/* cut stays false; loop will exit cleanly. */
|
||||||
}
|
}
|
||||||
/* else: complete sequence (expected==1 ASCII, or cont+1==expected)
|
|
||||||
* — nothing to do. */
|
|
||||||
} else {
|
|
||||||
/* Every byte was a continuation byte with no lead: discard all. */
|
|
||||||
u[0] = '\0';
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user