From bdc74f5fd266bcc9041359d2a53863f0e84cbf32 Mon Sep 17 00:00:00 2001 From: Martino Ferrari Date: Mon, 17 Aug 2026 00:06:45 +0200 Subject: [PATCH] StreamHub: loop UTF-8 tail repair to match Go CalConfig.Normalise() The single-pass repair left invalid bytes when the candidate lead byte had class 0 (illegal 0xF8-0xFF bytes, or a bare continuation byte reached after the 3-byte backward-scan cap). Convert to a loop with a `cut` flag mirroring Go's loop: each iteration either makes no cut (exits) or strictly reduces ulen by >= 1 byte (terminates in <= 16 iterations). Also treat expected==0 as a cut target, matching Go's behaviour of stripping any byte that decodes as an invalid one-byte sequence. Co-Authored-By: Claude Sonnet 4.6 --- .superpowers/sdd/task-4-report.md | 49 +++++++++++++ Source/Applications/StreamHub/StreamHub.cpp | 77 +++++++++++++-------- 2 files changed, 96 insertions(+), 30 deletions(-) diff --git a/.superpowers/sdd/task-4-report.md b/.superpowers/sdd/task-4-report.md index 196f55b..dfe6f3c 100644 --- a/.superpowers/sdd/task-4-report.md +++ b/.superpowers/sdd/task-4-report.md @@ -189,3 +189,52 @@ Overall: ALL PASS ``` All required cases verified: `"Ω"` survives unchanged, `"µs"` survives unchanged, 20-byte ASCII truncates to 16, a cut mid-rune truncates to the last complete rune, and a 16-byte string ending exactly on a complete multi-byte rune is untouched. + +## Fix round 3 + +### Change + +Converted the single-pass UTF-8 tail repair inside the `tlen > kMaxUnitLen` branch of `SetCalibrationEntry` into a loop that mirrors Go's `CalConfig.Normalise()` exactly. The new loop repeats the scan-and-cut until either no cut is made or `ulen` reaches zero. + +Two new cases are now handled that the old single pass missed: + +1. **Invalid lead byte class (`expected == 0`)** — bytes `0xF8`–`0xFF` (illegal in UTF-8) and bare continuation bytes found as the "candidate lead" after the backward scan hits its 3-byte cap. The old code left `expected = 0` and silently did nothing; the new code treats this the same as an incomplete sequence and cuts from that byte's position, setting `cut = true` so the loop continues. + +2. **Chains of continuation bytes longer than 3** — the backward scan caps at 3, so the candidate "lead" is itself a continuation byte. `expected` stays 0, the new path cuts it, and the loop re-runs until a valid lead (or empty string) is found. + +### Termination argument + +Each loop iteration either: (a) makes no cut → `cut` stays `false` → loop exits; or (b) strictly reduces `ulen` by at least 1 byte (the lead byte position `ulen - 1u - cont`, where `cont >= 0`). Because `ulen` is a `uint32` bounded below by zero and the guard `ulen > 0u` is checked on every iteration, the loop terminates after at most `kMaxUnitLen` (16) iterations. + +### Code diff (StreamHub.cpp, repair block) + +Old: single pass, no loop, `expected == 0` → silent no-op. +New: `bool cut = true; while (cut && ulen > 0u)` wraps the entire scan; `expected == 0` now sets `cut = true` and reduces `ulen`. + +### Standalone check output + +``` +g++ -std=c++98 -Wall -Werror -o /tmp/repair_test /tmp/repair_test.cpp && /tmp/repair_test + +PASS Omega untouched +PASS micros untouched +PASS 20 ASCII -> 16 +PASS mid-rune cut +PASS exact 16 complete rune +PASS UFFFD tail survives +PASS 20 continuation bytes -> empty +PASS illegal 0xF8 lead dropped + +All tests PASSED +``` + +### Build and test output + +``` +make -f Makefile.gcc apps → StreamHub.ex linked successfully (0 errors) + +./Build/x86-linux/GTest/MainGTest.ex +132 tests from 12 test cases ran. +PASSED: 127 +FAILED: 5 (UDPStreamerGTest multicast — pre-existing, machine-level issue; expected baseline 127/132 or 128/132) +``` diff --git a/Source/Applications/StreamHub/StreamHub.cpp b/Source/Applications/StreamHub/StreamHub.cpp index ba0251c..c7a57a4 100644 --- a/Source/Applications/StreamHub/StreamHub.cpp +++ b/Source/Applications/StreamHub/StreamHub.cpp @@ -791,38 +791,55 @@ bool StreamHub::SetCalibrationEntry(const char *source, const char *signal, u[kMaxUnitLen] = '\0'; uint32 ulen = kMaxUnitLen; - /* Scan back over continuation bytes (up to 3). */ - uint32 cont = 0u; - while ((cont < 3u) && (cont < ulen)) { - const unsigned char b = - static_cast(u[ulen - 1u - cont]); - if ((b & 0xC0u) == 0x80u) { - cont++; - } else { - break; - } - } + /* Repair any split or invalid rune at the tail. + * Mirror Go's loop: keep stripping until the tail is valid or empty. + * Each iteration either makes no cut (loop exits) or strictly reduces + * ulen by at least 1 byte, so termination is guaranteed. */ + bool cut = true; + while (cut && (ulen > 0u)) { + cut = false; - /* The byte at index ulen-1-cont is the candidate lead byte. */ - if (cont < ulen) { - const unsigned char lead = - static_cast(u[ulen - 1u - cont]); - uint32 expected = 0u; - if ((lead & 0x80u) == 0x00u) { expected = 1u; } - else if ((lead & 0xE0u) == 0xC0u) { expected = 2u; } - else if ((lead & 0xF0u) == 0xE0u) { expected = 3u; } - else if ((lead & 0xF8u) == 0xF0u) { expected = 4u; } - /* bytes present in the sequence = cont + 1 (the lead itself) */ - if ((expected > 1u) && ((cont + 1u) < expected)) { - /* Incomplete multi-byte sequence: drop from the lead byte. */ - ulen = ulen - 1u - cont; - u[ulen] = '\0'; + /* Scan back over continuation bytes (up to 3). */ + uint32 cont = 0u; + while ((cont < 3u) && (cont < ulen)) { + const unsigned char b = + static_cast(u[ulen - 1u - cont]); + if ((b & 0xC0u) == 0x80u) { + cont++; + } else { + break; + } + } + + /* The byte at index ulen-1-cont is the candidate lead byte. */ + if (cont < ulen) { + const unsigned char lead = + static_cast(u[ulen - 1u - cont]); + uint32 expected = 0u; + if ((lead & 0x80u) == 0x00u) { expected = 1u; } + else if ((lead & 0xE0u) == 0xC0u) { expected = 2u; } + else if ((lead & 0xF0u) == 0xE0u) { expected = 3u; } + else if ((lead & 0xF8u) == 0xF0u) { expected = 4u; } + /* expected==0: lead byte is not a valid UTF-8 lead class + * (0xF8-0xFF or a bare continuation); drop it too, like Go. */ + if (expected == 0u) { + /* Invalid lead byte: strip from that position. */ + ulen = ulen - 1u - cont; + u[ulen] = '\0'; + cut = true; + } else if ((cont + 1u) < expected) { + /* Incomplete multi-byte sequence: drop from the lead byte. */ + ulen = ulen - 1u - cont; + u[ulen] = '\0'; + cut = true; + } + /* else: complete sequence — nothing to do, loop exits. */ + } else { + /* Every byte was a continuation byte with no lead: discard all. */ + ulen = 0u; + u[0] = '\0'; + /* cut stays false; loop will exit cleanly. */ } - /* else: complete sequence (expected==1 ASCII, or cont+1==expected) - * — nothing to do. */ - } else { - /* Every byte was a continuation byte with no lead: discard all. */ - u[0] = '\0'; } } }