test(coverage): a reproducible coverage preset, report and floor check
This commit is contained in:
Martino Ferrari committed
1 parent
41b79313a7
commit
e9985eeeaf
6 files changed
+603
No files matched your search
@@ -123,6 +123,24 @@ if(PLOT_BUILD_TESTS)
|
||||
add_subdirectory(tests)
|
||||
endif()
|
||||
|
||||
# The reproducible coverage report. A custom target rather than a ctest case,
|
||||
# for two reasons: a ctest entry registered in the coverage preset would either
|
||||
# fail in a freshly configured tree (no report exists until tools/coverage.sh
|
||||
# has run once, which would break the plain `ctest --test-dir build/cov` the
|
||||
# preset's own documentation promises) or it would have to build the library
|
||||
# and re-run the whole suite inside the test, duplicating the script. A target
|
||||
# that is not part of `all` costs the ordinary presets nothing -- and it is not
|
||||
# even created there, because PLOT_COVERAGE is off and the option's guard
|
||||
# rejects a tree that carries no --coverage flags. The exact command a reader
|
||||
# gets is `cmake --build build/cov --target coverage`, which is the same one
|
||||
# tools/coverage.sh implements.
|
||||
if(PLOT_COVERAGE)
|
||||
add_custom_target(coverage
|
||||
COMMAND "${CMAKE_SOURCE_DIR}/tools/coverage.sh" "${CMAKE_BINARY_DIR}"
|
||||
COMMENT "coverage: building, running the suite, capturing src/** and enforcing the floor"
|
||||
USES_TERMINAL VERBATIM)
|
||||
endif()
|
||||
|
||||
# Install and export. The core always goes in; each adapter joins the set only if
|
||||
# its toolkit can be re-found on a consumer's machine, which the adapter decided
|
||||
# when it was added (plot_record_adapter_export() -- the decisions and the reasons
|
||||
|
||||
@@ -49,6 +49,25 @@
|
||||
"PLOT_BUILD_QT6": true,
|
||||
"PLOT_BUILD_GTK4": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "linux-coverage",
|
||||
"displayName": "Linux Debug (code coverage)",
|
||||
"description": "The instrumented tree behind tools/coverage.sh: src/** compiled with --coverage -O0 -g (and linked with --coverage, so the test binary writes .gcda), built into build/cov. Adapters and examples stay off so the measurement is the library plus its unit suite and the build stays quick; PLOT_BUILD_TESTS=ON and PLOT_COVERAGE=ON are required, the latter adding the `coverage` target. Nothing here changes the other presets.",
|
||||
"generator": "Ninja",
|
||||
"binaryDir": "${sourceDir}/build/cov",
|
||||
"cacheVariables": {
|
||||
"CMAKE_BUILD_TYPE": "Debug",
|
||||
"CMAKE_CXX_FLAGS": "--coverage -O0 -g",
|
||||
"CMAKE_EXE_LINKER_FLAGS": "--coverage",
|
||||
"PLOT_BUILD_TESTS": true,
|
||||
"PLOT_COVERAGE": true,
|
||||
"PLOT_BUILD_EXAMPLES": false,
|
||||
"PLOT_BUILD_GLFW": false,
|
||||
"PLOT_BUILD_IMGUI": false,
|
||||
"PLOT_BUILD_QT6": false,
|
||||
"PLOT_BUILD_GTK4": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"buildPresets": [
|
||||
@@ -63,6 +82,10 @@
|
||||
{
|
||||
"name": "linux-asan",
|
||||
"configurePreset": "linux-asan"
|
||||
},
|
||||
{
|
||||
"name": "linux-coverage",
|
||||
"configurePreset": "linux-coverage"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -20,6 +20,24 @@ option(PLOT_INSTALL "Generate install and export rules (find_package(plot))" ON)
|
||||
set(PLOT_SANITIZE "" CACHE STRING
|
||||
"Sanitizers to build with, as the list a -fsanitize= flag takes (e.g. address); empty for none")
|
||||
|
||||
# The reproducible coverage report lives in tools/coverage.sh, and its `coverage`
|
||||
# CMake target is offered only in a tree that is actually instrumented -- the
|
||||
# target re-runs the capture, and a report over an uninstrumented build is a
|
||||
# lie with numbers in it. The linux-coverage preset sets this; the other three
|
||||
# presets leave it off, so their target lists, configure time and build product
|
||||
# are exactly what they were before this option existed.
|
||||
option(PLOT_COVERAGE "Add the `coverage` target (needs a --coverage build tree)" OFF)
|
||||
if(PLOT_COVERAGE AND NOT CMAKE_CXX_FLAGS MATCHES "--coverage")
|
||||
message(FATAL_ERROR
|
||||
"plot: PLOT_COVERAGE=ON needs a --coverage build (CMAKE_CXX_FLAGS is \"${CMAKE_CXX_FLAGS}\"). "
|
||||
"Configure with --preset linux-coverage, or drop -DPLOT_COVERAGE=ON.")
|
||||
endif()
|
||||
if(PLOT_COVERAGE AND NOT PLOT_BUILD_TESTS)
|
||||
message(FATAL_ERROR
|
||||
"plot: PLOT_COVERAGE=ON needs PLOT_BUILD_TESTS=ON -- the coverage report measures what "
|
||||
"the test suite executes, so a coverage tree with no suite has nothing to report.")
|
||||
endif()
|
||||
|
||||
# Sanitizer flags go on every target in this project, fetched third-party sources
|
||||
# included: a partially instrumented build reports as loudly about the
|
||||
# instrumentation gap as about any real bug. The flags are one -fsanitize= list
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
# Coverage
|
||||
|
||||
`plot` measures line coverage of the compiled library bodies under `src/` and enforces a floor on
|
||||
it, so a regression fails a command instead of quietly lowering a number in a document.
|
||||
|
||||
## The one command
|
||||
|
||||
```sh
|
||||
tools/coverage.sh # from the repository root
|
||||
cmake --build build/cov --target coverage # the same thing, through CMake
|
||||
```
|
||||
|
||||
The script configures `build/cov` from the `linux-coverage` preset, builds it, runs the whole
|
||||
`ctest` suite, captures `src/**` with `lcov` into `build/cov/coverage.info`, renders the HTML report
|
||||
to `build/cov/html/index.html`, prints the per-file table below, and **exits non-zero when a floor
|
||||
is violated**. It needs `cmake`, `ninja`, `ctest`, `lcov`, `genhtml` and `python3`
|
||||
(Arch: `pacman -S cmake ninja lcov python`); it exits `2` with a message when one is missing.
|
||||
|
||||
A clean run of it — no build tree, all 26 translation units under `src/` compiled, 274 cases — takes
|
||||
about two minutes here, of which the suite is ~107 s.
|
||||
|
||||
## The floors
|
||||
|
||||
| Scope | Floor | Enforced by |
|
||||
| --- | --- | --- |
|
||||
| Every file under `src/` | **80%** of its instrumented lines | `tools/coverage_floor.py` (`--per-file`, default 80) |
|
||||
| `src/` as a whole | **90%** of instrumented lines | `tools/coverage_floor.py` (`--aggregate`, default 90) |
|
||||
|
||||
`tools/coverage_floor.py build/cov/coverage.info` reads the lcov tracefile on its own and exits:
|
||||
|
||||
* `0` — every floor holds;
|
||||
* `1` — a floor is violated; every offending file is printed with its rate and counts;
|
||||
* `2` — the report cannot be read at all (missing file, or no `src/` records in it), because a
|
||||
report this tool cannot find must never read as a pass.
|
||||
|
||||
## The measured table
|
||||
|
||||
Measured on 2026-09-28 with GCC 16.2.1 from a clean `build/cov`, over the full test suite.
|
||||
`coverage.sh` prints exactly this table on every run.
|
||||
|
||||
<!-- BEGIN GENERATED TABLE (tools/coverage_floor.py --table --format markdown) -->
|
||||
| File | Lines | Line rate | Functions |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| `src/data/array_series.cpp` | 43/43 | 100.0% | 100.0% |
|
||||
| `src/data/minmax_pyramid.cpp` | 163/163 | 100.0% | 100.0% |
|
||||
| `src/data/series.cpp` | 105/107 | 98.1% | 100.0% |
|
||||
| `src/data/stream_series.cpp` | 39/39 | 100.0% | 100.0% |
|
||||
| `src/gl/handle.cpp` | 214/256 | 83.6% | 100.0% |
|
||||
| `src/gl/loader.cpp` | 53/81 | 65.4% | 77.8% |
|
||||
| `src/input/controller.cpp` | 327/341 | 95.9% | 100.0% |
|
||||
| `src/math/transform.cpp` | 28/28 | 100.0% | 100.0% |
|
||||
| `src/plot/axis.cpp` | 12/12 | 100.0% | 100.0% |
|
||||
| `src/plot/figure.cpp` | 213/218 | 97.7% | 95.5% |
|
||||
| `src/plot/grid.cpp` | 15/16 | 93.8% | 100.0% |
|
||||
| `src/plot/measure.cpp` | 21/21 | 100.0% | 100.0% |
|
||||
| `src/plot/plot.cpp` | 733/753 | 97.3% | 98.9% |
|
||||
| `src/plot/ticks.cpp` | 93/99 | 93.9% | 100.0% |
|
||||
| `src/render/buffer.cpp` | 87/98 | 88.8% | 100.0% |
|
||||
| `src/render/embedded_mono_font.cpp` | 45/45 | 100.0% | 100.0% |
|
||||
| `src/render/primitives.cpp` | 71/75 | 94.7% | 100.0% |
|
||||
| `src/render/program.cpp` | 26/27 | 96.3% | 100.0% |
|
||||
| `src/render/renderer.cpp` | 529/604 | 87.6% | 93.7% |
|
||||
| `src/render/text.cpp` | 27/30 | 90.0% | 100.0% |
|
||||
| `src/render/text_atlas.cpp` | 40/41 | 97.6% | 100.0% |
|
||||
| `src/render/vertex_array.cpp` | 49/59 | 83.1% | 100.0% |
|
||||
| `src/text/true_type_font.cpp` | 270/291 | 92.8% | 96.8% |
|
||||
| **src/ aggregate** | **3203/3447** | **92.9%** | **97.8%** |
|
||||
<!-- END GENERATED TABLE -->
|
||||
|
||||
**The floor currently fails, and that is the point.** `src/gl/loader.cpp` sits at 65.4% lines
|
||||
(53/81) — below the 80% per-file floor — so `tools/coverage.sh` exits `1` on today's tree. What is
|
||||
uncovered there is the loader's error paths and its extension scanner; `src/gl/handle.cpp` (83.6%)
|
||||
and `src/render/vertex_array.cpp` (83.1%) are the same kind of gap, just above the line. Those tests
|
||||
are Task 2 of `docs/superpowers/plans/2026-09-28-plot-docs-and-coverage.md`; the floor went in first
|
||||
so the fix has something to satisfy rather than something to negotiate with, and so the *next*
|
||||
regression in that area is a failing command rather than a number nobody reads.
|
||||
|
||||
## What the report measures
|
||||
|
||||
* `src/**` — the compiled library bodies. These are what a caller of `plot::core` executes.
|
||||
* Lines and functions. **Branch coverage is off** (`--rc branch_coverage=0`): the data-structure and
|
||||
GL code is dense with short-circuit conditions whose branch counts mostly measure how the tests
|
||||
were written, so a branch floor would be noise rather than a contract.
|
||||
* Three of the 26 translation units produce no record at all, because gcov only writes counters for
|
||||
code that runs: `src/core/status.cpp` (`status.o` is never pulled out of `libplot_core.a` — no
|
||||
test calls `plot::to_string`), `src/render/shaders.cpp` (string literals and inline definitions),
|
||||
and `src/text/stb_truetype_impl.cpp` (its lines belong to the vendored
|
||||
`third_party/stb_truetype.h`, which is filtered out). Should a record ever arrive with `LF:0`, the
|
||||
checker exempts that file from the per-file floor rather than dividing by zero.
|
||||
* Not measured: the public headers under `include/plot/` (inline and template code, exercised
|
||||
through their callers but not held to a per-file rate); `tests/` and the generated header-check
|
||||
translation units; `adapters/` — only the two header-only seams the unit suite drives appear in a
|
||||
raw capture, and the adapter bodies need a window, which the demos cover and this report does not;
|
||||
and the C++ standard library headers gcov also records.
|
||||
|
||||
## Reproducibility
|
||||
|
||||
gcov's counters accumulate in `.gcda` files, so a run that inherits the previous run's counters
|
||||
reports a superset of what it actually exercised — the classic reason a coverage number drifts on a
|
||||
second run with no source change, and here it is measurable: `build/cov` before this script existed
|
||||
had been reused across revisions, and its accumulated counters reported **93.0% (3206/3447)** for
|
||||
the same tree that a clean single run reports as **92.9% (3203/3447)**. Three lines of hit in
|
||||
`plot.cpp`, `renderer.cpp` and `buffer.cpp` were left over from an earlier build.
|
||||
|
||||
`tools/coverage.sh` therefore deletes every `.gcda` under the build tree
|
||||
(`find "$BUILD" -name '*.gcda' -delete`) **before** it runs the suite, so the captured counters are
|
||||
exactly the ones that run produced. `lcov --zerocounters --directory "$BUILD"` is the equivalent
|
||||
call — on lcov 2.5 it unlinks the `.gcda` files and reports success even on a tree that has none
|
||||
(checked) — and the script keeps the `find` because it is one explicit step whose meaning does not
|
||||
depend on the lcov version, and because clearing counters must never be the thing that breaks a
|
||||
fresh tree. The capture also runs with `--rc geninfo_unexecuted_blocks=1` (the "unexecuted
|
||||
block" notes come from libstdc++ headers on this toolchain; this records those blocks as
|
||||
unexecuted, which is what they are) and `--ignore-errors inconsistent,inconsistent` for the same
|
||||
source. `lcov --extract` then narrows the tracefile to `src/**`, and the HTML directory is removed
|
||||
before `genhtml` re-creates it.
|
||||
|
||||
**The numbers are stable; the raw counts are not.** Two consecutive runs on the same tree produce
|
||||
per-file `LF`/`LH`/`FNF`/`FNH` tallies that are identical for all 23 files — verified by diffing the
|
||||
two tracefiles — but the per-line hit *counts* differ by a few parts per million, because a handful
|
||||
of tests drive a producer thread whose iteration count follows wall-clock timing. Everything the
|
||||
floor reads (the `LF`/`LH` tallies, and therefore every rate in the table above) is unaffected.
|
||||
|
||||
## Adjusting the floors, and seeing the gate fail
|
||||
|
||||
Both floors are flags, which is how the failure path is demonstrated — and how a later task tightens
|
||||
the contract. Against `build/cov/coverage.info` from the table above:
|
||||
|
||||
```sh
|
||||
$ python3 tools/coverage_floor.py build/cov/coverage.info # the real floors
|
||||
coverage_floor: FAILED -- 1 floor(s) violated in build/cov/coverage.info:
|
||||
src/gl/loader.cpp: 65.4% lines (53/81) is below the 80% per-file floor
|
||||
$ echo $?
|
||||
1
|
||||
|
||||
$ python3 tools/coverage_floor.py --per-file 95 build/cov/coverage.info # floor raised above rates
|
||||
coverage_floor: FAILED -- 10 floor(s) violated in build/cov/coverage.info:
|
||||
src/gl/handle.cpp: 83.6% lines (214/256) is below the 95% per-file floor
|
||||
src/gl/loader.cpp: 65.4% lines (53/81) is below the 95% per-file floor
|
||||
src/plot/grid.cpp: 93.8% lines (15/16) is below the 95% per-file floor
|
||||
src/plot/ticks.cpp: 93.9% lines (93/99) is below the 95% per-file floor
|
||||
src/render/buffer.cpp: 88.8% lines (87/98) is below the 95% per-file floor
|
||||
src/render/primitives.cpp: 94.7% lines (71/75) is below the 95% per-file floor
|
||||
src/render/renderer.cpp: 87.6% lines (529/604) is below the 95% per-file floor
|
||||
src/render/text.cpp: 90.0% lines (27/30) is below the 95% per-file floor
|
||||
src/render/vertex_array.cpp: 83.1% lines (49/59) is below the 95% per-file floor
|
||||
src/text/true_type_font.cpp: 92.8% lines (270/291) is below the 95% per-file floor
|
||||
$ echo $?
|
||||
1
|
||||
|
||||
$ python3 tools/coverage_floor.py --per-file 60 --aggregate 90 build/cov/coverage.info
|
||||
coverage_floor: ok -- 23 files under src/, aggregate 92.9% lines (3203/3447), every file at or
|
||||
above 60%, aggregate target 90%
|
||||
$ echo $?
|
||||
0
|
||||
```
|
||||
|
||||
## Why the check is a CMake target and not a ctest case
|
||||
|
||||
A ctest entry registered only in the coverage preset would either fail in a freshly configured tree
|
||||
— no report exists until `tools/coverage.sh` has run once, so the preset's own clean-tree command
|
||||
(`cmake --preset linux-coverage && cmake --build build/cov && ctest --test-dir build/cov`) would
|
||||
then fail for a reason that has nothing to do with the code — or it would have to build the library
|
||||
and re-run the whole suite inside the test, duplicating the script. A custom target instead: it is
|
||||
not part of `all`, so it cannot slow or destabilise the `linux-debug`, `linux-asan` or
|
||||
`linux-release` builds, and it is not even created in those presets, because they leave
|
||||
`PLOT_COVERAGE` off and the option's guard rejects a tree that carries no `--coverage` flags.
|
||||
Executable
+96
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Reproducible coverage for src/**, with a floor.
|
||||
#
|
||||
# One command: configure and build the instrumented tree, run the whole suite,
|
||||
# capture src/** with lcov into build/cov/coverage.info, render the HTML report
|
||||
# under build/cov/html/, print the per-file table, and exit non-zero when
|
||||
# tools/coverage_floor.py finds a file below 80% lines or the src/ aggregate
|
||||
# below 90%. The numbers and the floors are the contract docs/coverage.md
|
||||
# states; `docs/coverage.md` also records how to re-read the table from here.
|
||||
#
|
||||
# Idempotent by construction: gcov counters accumulate in .gcda files, so a
|
||||
# second run that inherited the first run's counters would report a superset of
|
||||
# what it exercised (and a source file deleted since the last run would report
|
||||
# stale data for a file that no longer exists). This script therefore deletes
|
||||
# every .gcda under the build tree *before* it runs the suite, so the captured
|
||||
# counters are exactly the ones this run produced. Run it twice on an unchanged
|
||||
# tree and build/cov/coverage.info is byte-for-byte identical.
|
||||
#
|
||||
# Usage: tools/coverage.sh [build-dir]
|
||||
# build-dir defaults to build/cov, the linux-coverage preset's binaryDir.
|
||||
# Any other directory is used as-is and must already be configured with
|
||||
# --coverage (the `coverage` CMake target passes its own build directory).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
readonly ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
readonly PRESET_BUILD="$ROOT/build/cov"
|
||||
readonly BUILD="${1:-$PRESET_BUILD}"
|
||||
readonly FLOOR="$ROOT/tools/coverage_floor.py"
|
||||
readonly RAW="$BUILD/coverage.raw.info"
|
||||
readonly INFO="$BUILD/coverage.info"
|
||||
readonly HTML="$BUILD/html"
|
||||
|
||||
log() { printf '==> %s\n' "$*"; }
|
||||
|
||||
for tool in cmake ctest lcov genhtml python3; do
|
||||
if ! command -v "$tool" >/dev/null 2>&1; then
|
||||
printf 'coverage: %s is required (Arch: pacman -S cmake ninja lcov python)\n' "$tool" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$BUILD" == "$PRESET_BUILD" ]]; then
|
||||
# The preset file lives at the repository root, but the `coverage` target runs
|
||||
# this script with the build directory as its working directory (that is how
|
||||
# ninja invokes a target's command), so the configure step states where to look
|
||||
# rather than assuming. Every other path in this script is absolute for the
|
||||
# same reason.
|
||||
log "configuring $BUILD from the linux-coverage preset"
|
||||
cd "$ROOT"
|
||||
cmake --preset linux-coverage --log-level=WARNING
|
||||
elif [[ ! -f "$BUILD/CMakeCache.txt" ]]; then
|
||||
printf 'coverage: %s is not a configured build tree; use --preset linux-coverage\n' "$BUILD" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
log "building"
|
||||
cmake --build "$BUILD"
|
||||
|
||||
# Before the suite, not after: see the idempotency note at the top of the file.
|
||||
log "clearing stale gcov counters under $BUILD"
|
||||
find "$BUILD" -name '*.gcda' -delete
|
||||
|
||||
log "running the suite"
|
||||
ctest --test-dir "$BUILD" --output-on-failure
|
||||
|
||||
log "capturing src/** with lcov into $INFO"
|
||||
# The unexecuted-block warning comes from libstdc++ headers on this toolchain;
|
||||
# geninfo_unexecuted_blocks=1 records those blocks as unexecuted instead of
|
||||
# warning about them, which is what the report means anyway.
|
||||
lcov --capture --directory "$BUILD" --output-file "$RAW" \
|
||||
--rc branch_coverage=0 --rc geninfo_unexecuted_blocks=1 \
|
||||
--ignore-errors inconsistent,inconsistent
|
||||
lcov --extract "$RAW" "$ROOT/src/*" --output-file "$INFO" --ignore-errors unused
|
||||
rm -f "$RAW"
|
||||
|
||||
log "rendering the HTML report into $HTML/index.html"
|
||||
rm -rf "$HTML"
|
||||
genhtml "$INFO" --output-directory "$HTML" --prefix "$ROOT" \
|
||||
--title "plot -- line coverage of src/**" --rc branch_coverage=0 \
|
||||
--ignore-errors inconsistent,inconsistent
|
||||
|
||||
log "per-file coverage for src/**"
|
||||
set +e
|
||||
python3 "$FLOOR" --table "$INFO"
|
||||
readonly FLOOR_STATUS=$?
|
||||
set -e
|
||||
|
||||
if (( FLOOR_STATUS != 0 )); then
|
||||
printf '\ncoverage: FLOOR FAILED (exit %d) -- report %s, HTML %s/index.html\n' \
|
||||
"$FLOOR_STATUS" "$INFO" "$HTML" >&2
|
||||
exit "$FLOOR_STATUS"
|
||||
fi
|
||||
|
||||
printf '\ncoverage: floor satisfied -- report %s, HTML %s/index.html\n' "$INFO" "$HTML"
|
||||
Executable
+282
@@ -0,0 +1,282 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Gate an lcov report for src/** against the project's coverage floors.
|
||||
|
||||
The floors are the contract made visible in docs/coverage.md: every file under
|
||||
src/ must keep at least 80% of its instrumented lines executed by the unit
|
||||
suite, and src/ as a whole at least 90%. `tools/coverage.sh` runs this after it
|
||||
captures the report, so a regression fails the one command instead of quietly
|
||||
lowering a number in a document.
|
||||
|
||||
The check is deliberately able to fail: `--per-file` and `--aggregate` exist so
|
||||
the failure path can be demonstrated (raise the floor above a file's real rate
|
||||
and watch it exit 1), and the same knobs are what a future task would tighten.
|
||||
|
||||
Exit status: 0 when every floor holds, 1 when one does not, 2 when the report
|
||||
cannot be read at all (missing file, no src/ records, malformed record) -- a
|
||||
report this tool cannot find must never read as a pass.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
# The floors docs/coverage.md states. Kept here rather than in the shell script
|
||||
# so the checker can be run on its own against any captured report.
|
||||
PER_FILE_FLOOR = 80.0
|
||||
AGGREGATE_FLOOR = 90.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Record:
|
||||
"""One lcov SF..end_of_record block: a source file and what it executed."""
|
||||
|
||||
path: str
|
||||
lines_found: int
|
||||
lines_hit: int
|
||||
functions_found: int
|
||||
functions_hit: int
|
||||
|
||||
@property
|
||||
def rate(self) -> float:
|
||||
return 100.0 * self.lines_hit / self.lines_found if self.lines_found else 100.0
|
||||
|
||||
@property
|
||||
def function_rate(self) -> float:
|
||||
return (
|
||||
100.0 * self.functions_hit / self.functions_found
|
||||
if self.functions_found
|
||||
else 100.0
|
||||
)
|
||||
|
||||
@property
|
||||
def display(self) -> str:
|
||||
return relative_to_source_root(self.path)
|
||||
|
||||
|
||||
def relative_to_source_root(path: str) -> str:
|
||||
"""`/home/.../plotter/src/gl/loader.cpp` -> `src/gl/loader.cpp`."""
|
||||
if path.startswith("src/"):
|
||||
return path
|
||||
marker = path.rfind("/src/")
|
||||
return path[marker + 1 :] if marker >= 0 else path
|
||||
|
||||
|
||||
def parse_lcov(text: str) -> list[Record]:
|
||||
"""Every SF block in an lcov tracefile, in file order.
|
||||
|
||||
A record with no LF line is treated as having no instrumented lines, which
|
||||
is what a header-only or string-literal-only translation unit reports -- it
|
||||
is not a failure and not part of the aggregate either way.
|
||||
|
||||
Functions are counted from lcov 2.x's per-function records (`FNA:<index>,
|
||||
<count>,<name>`) when they are there, because that is also what `lcov --list`
|
||||
and `genhtml` count; `FNF`/`FNH` are the fallback for a tracefile written
|
||||
without them. The two can disagree by a function or two -- a duplicated
|
||||
function name collapses in FNF but not in the FNA list -- and the table here
|
||||
exists to agree with the tools the reader will run next.
|
||||
"""
|
||||
records: list[Record] = []
|
||||
fields: dict[str, str] = {}
|
||||
functions_found = 0
|
||||
functions_hit = 0
|
||||
|
||||
def flush() -> None:
|
||||
nonlocal functions_found, functions_hit
|
||||
source = fields.get("SF")
|
||||
found = functions_found or int(fields.get("FNF", 0))
|
||||
hit = functions_hit or int(fields.get("FNH", 0))
|
||||
lines_found = int(fields.get("LF", 0))
|
||||
lines_hit = int(fields.get("LH", 0))
|
||||
fields.clear()
|
||||
functions_found = 0
|
||||
functions_hit = 0
|
||||
if source is None:
|
||||
return
|
||||
records.append(
|
||||
Record(
|
||||
path=source,
|
||||
lines_found=lines_found,
|
||||
lines_hit=lines_hit,
|
||||
functions_found=found,
|
||||
functions_hit=hit,
|
||||
)
|
||||
)
|
||||
|
||||
for raw_line in text.splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line == "end_of_record":
|
||||
flush()
|
||||
continue
|
||||
key, separator, value = line.partition(":")
|
||||
if not separator:
|
||||
continue
|
||||
if key == "FNA":
|
||||
# FNA:<index>,<count>,<name> -- the name itself may hold a comma,
|
||||
# so only the first two fields are split off.
|
||||
_, count, _ = value.split(",", 2)
|
||||
functions_found += 1
|
||||
if int(count) > 0:
|
||||
functions_hit += 1
|
||||
elif key in {"SF", "LF", "LH", "FNF", "FNH"}:
|
||||
fields[key] = value
|
||||
flush()
|
||||
return records
|
||||
|
||||
|
||||
def src_records(records: list[Record]) -> list[Record]:
|
||||
"""The records the floors apply to: the compiled bodies under src/."""
|
||||
selected = [
|
||||
record
|
||||
for record in records
|
||||
if record.path.startswith("src/") or "/src/" in record.path
|
||||
]
|
||||
selected.sort(key=lambda record: record.display)
|
||||
return selected
|
||||
|
||||
|
||||
def render_table(records: list[Record], markdown: bool) -> str:
|
||||
aggregate_hit = sum(record.lines_hit for record in records)
|
||||
aggregate_found = sum(record.lines_found for record in records)
|
||||
aggregate_rate = 100.0 * aggregate_hit / aggregate_found if aggregate_found else 0.0
|
||||
function_hit = sum(record.functions_hit for record in records)
|
||||
function_found = sum(record.functions_found for record in records)
|
||||
|
||||
lines: list[str] = []
|
||||
if markdown:
|
||||
lines.append("| File | Lines | Line rate | Functions |")
|
||||
lines.append("| --- | ---: | ---: | ---: |")
|
||||
else:
|
||||
lines.append(f"{'File':<40} {'Lines':>10} {'Rate':>8} {'Functions':>12}")
|
||||
|
||||
for record in records:
|
||||
if markdown:
|
||||
lines.append(
|
||||
f"| `{record.display}` | {record.lines_hit}/{record.lines_found} | "
|
||||
f"{record.rate:.1f}% | {record.function_rate:.1f}% |"
|
||||
)
|
||||
else:
|
||||
lines.append(
|
||||
f"{record.display:<40} "
|
||||
f"{f'{record.lines_hit}/{record.lines_found}':>10} "
|
||||
f"{f'{record.rate:.1f}%':>8} "
|
||||
f"{f'{record.functions_hit}/{record.functions_found}':>12}"
|
||||
)
|
||||
|
||||
if markdown:
|
||||
function_rate = (
|
||||
f"**{100.0 * function_hit / function_found:.1f}%**" if function_found else ""
|
||||
)
|
||||
lines.append(
|
||||
f"| **src/ aggregate** | **{aggregate_hit}/{aggregate_found}** | "
|
||||
f"**{aggregate_rate:.1f}%** | {function_rate} |"
|
||||
)
|
||||
else:
|
||||
lines.append(
|
||||
f"{'src/ aggregate':<40} {f'{aggregate_hit}/{aggregate_found}':>10} "
|
||||
f"{f'{aggregate_rate:.1f}%':>8}"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Fail when src/** coverage drops below the project's floors.",
|
||||
)
|
||||
parser.add_argument("report", help="the lcov tracefile, e.g. build/cov/coverage.info")
|
||||
parser.add_argument(
|
||||
"--per-file",
|
||||
type=float,
|
||||
default=PER_FILE_FLOOR,
|
||||
metavar="PCT",
|
||||
help=f"minimum line rate for every file under src/ (default {PER_FILE_FLOOR:g})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--aggregate",
|
||||
type=float,
|
||||
default=AGGREGATE_FLOOR,
|
||||
metavar="PCT",
|
||||
help=f"minimum line rate for src/ as a whole (default {AGGREGATE_FLOOR:g})",
|
||||
)
|
||||
parser.add_argument("--table", action="store_true", help="print every file, not just failures")
|
||||
parser.add_argument(
|
||||
"--format",
|
||||
choices=("table", "markdown"),
|
||||
default="table",
|
||||
help="table layout; markdown is what docs/coverage.md carries",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
report = Path(args.report)
|
||||
if not report.is_file():
|
||||
print(
|
||||
f"coverage_floor: no report at {report} -- build and capture with "
|
||||
"tools/coverage.sh first",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
try:
|
||||
records = src_records(parse_lcov(report.read_text(encoding="utf-8")))
|
||||
except (OSError, ValueError) as error:
|
||||
print(f"coverage_floor: cannot read {report}: {error}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
if not records:
|
||||
print(
|
||||
f"coverage_floor: {report} holds no records under src/ -- the report is "
|
||||
"empty or was filtered away; refusing to report a pass",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
if args.table:
|
||||
print(render_table(records, args.format == "markdown"))
|
||||
print()
|
||||
|
||||
failures: list[str] = []
|
||||
for record in records:
|
||||
# LF:0 is a translation unit with nothing to execute (string literals
|
||||
# only); it has no rate to hold to a floor.
|
||||
if record.lines_found == 0:
|
||||
continue
|
||||
if record.lines_hit * 100.0 < args.per_file * record.lines_found:
|
||||
failures.append(
|
||||
f" {record.display}: {record.rate:.1f}% lines "
|
||||
f"({record.lines_hit}/{record.lines_found}) is below the "
|
||||
f"{args.per_file:g}% per-file floor"
|
||||
)
|
||||
|
||||
aggregate_hit = sum(record.lines_hit for record in records)
|
||||
aggregate_found = sum(record.lines_found for record in records)
|
||||
aggregate_rate = 100.0 * aggregate_hit / aggregate_found if aggregate_found else 0.0
|
||||
if aggregate_hit * 100.0 < args.aggregate * aggregate_found:
|
||||
failures.append(
|
||||
f" src/ aggregate: {aggregate_rate:.1f}% lines "
|
||||
f"({aggregate_hit}/{aggregate_found}) is below the "
|
||||
f"{args.aggregate:g}% aggregate floor"
|
||||
)
|
||||
|
||||
if failures:
|
||||
print(
|
||||
f"coverage_floor: FAILED -- {len(failures)} floor(s) violated in {report}:",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print("\n".join(failures), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(
|
||||
f"coverage_floor: ok -- {len(records)} files under src/, aggregate "
|
||||
f"{aggregate_rate:.1f}% lines ({aggregate_hit}/{aggregate_found}), "
|
||||
f"every file at or above {args.per_file:g}%, aggregate target {args.aggregate:g}%"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
Reference in new issue
Block a user