fix(text): refuse an atlas that would overflow its own arithmetic, and clamp the cell before narrowing it

src/text/true_type_font.cpp

@@ -240,22 +240,40 @@ bool bake_face(const stbtt_fontinfo& info, f32 scale, std::span<const u32> cover
   // The cell: wide enough for the widest advance *and* the widest ink (the
   // generator's rule -- an overhanging glyph stays inside its own cell instead of
   // reaching into the next one), as tall as the face's ascent-to-descent extent,
-  // and never so short that a baked bitmap box hangs out of it.
-  const u32 cell_w = std::max<u32>(
-      1u,
-      static_cast<u32>(std::ceil(std::max(widest_advance, static_cast<f32>(rightmost)) - 1e-4f)));
+  // and never so short that a baked bitmap box hangs out of it. Each extent is
+  // clamped before it is narrowed to texels: a float-to-unsigned conversion of a
+  // negative value is undefined, and a face whose metrics or ink are degenerate
+  // enough to produce one must land on a one-texel cell, not on undefined
+  // behaviour.
+  const auto ceil_to_texels = [](f32 value) {
+    return static_cast<u32>(std::ceil(std::max(0.0f, value - 1e-4f)));
+  };
+  const u32 cell_w =
+      std::max<u32>(1u, ceil_to_texels(std::max(widest_advance, static_cast<f32>(rightmost))));
   i32 baseline = static_cast<i32>(std::floor(ascent_f));
   baseline = std::max(baseline, -highest);  // room for the tallest ink above the pen
   baseline = std::max(baseline, 0);
-  const u32 extent = static_cast<u32>(std::ceil(ascent_f + descent_f - 1e-4f));
   const u32 below = static_cast<u32>(std::max(0, baseline + lowest));
-  const u32 cell_h = std::max<u32>(1u, std::max(extent, below));
+  const u32 cell_h = std::max<u32>(1u, std::max(ceil_to_texels(ascent_f + descent_f), below));
+
+  // The atlas' width is where a caller's coverage list becomes a buffer size, so
+  // the product is formed in 64 bits and checked before it is narrowed: a set
+  // large enough to wrap it would otherwise allocate one buffer and index another,
+  // which is the one arithmetic error in this file that a host could reach by
+  // asking for too much rather than by handing over a broken font.
+  const u32 slots = static_cast<u32>(drawable.size()) + 1u;  // the fallback box is the last slot
+  const u64 atlas_w = static_cast<u64>(cell_w) * static_cast<u64>(slots);
+  constexpr u64 kMaxAtlasTexels = 1ull << 30;  // a gibibyte of R8 coverage
+  if (atlas_w > static_cast<u64>(0xFFFFFFFFu) ||
+      atlas_w * static_cast<u64>(cell_h) > kMaxAtlasTexels) {
+    return fail(st, ErrorCode::InvalidArgument,
+                "the requested coverage needs an atlas larger than this backend will hold");
+  }
 
   out->cell_w = cell_w;
   out->cell_h = cell_h;
   out->baseline = baseline;
-  const u32 slots = static_cast<u32>(drawable.size()) + 1u;  // the fallback box is the last slot
-  out->atlas_w = cell_w * slots;
+  out->atlas_w = static_cast<u32>(atlas_w);
   out->atlas_h = cell_h;
   out->atlas.assign(static_cast<std::size_t>(out->atlas_w) * out->atlas_h, 0u);
   out->codepoints.clear();