From dbebbd9803e593c2c44ac0b6ea8301a3600acd74 Mon Sep 17 00:00:00 2001 From: dovvnloading <157447210+dovvnloading@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:05:06 -0400 Subject: [PATCH] Measure everything that ships, and notice the 300-line cap filling up Two gates whose SCOPE was wrong rather than whose logic was. [tool.coverage.run].source was ["backend", "graphlink_plugins"], while [tool.setuptools] ships four packages and 23 loose root modules. Two shipped packages - provider_runtime and settings_store - and every root module, api_provider.py among them, sat outside the 85% floor entirely. A file nothing measures can regress to zero with the gate still green. Widening it takes the measured set from 18,763 statements to 21,606 and lands at 85.53%, so the floor holds unchanged. provider_runtime is the weak spot now inside it (ollama_scan 8%, llama_cpp_scan 10%, gemini_transport 29%) and that is the point: those numbers were invisible before, and a regression in them now counts. The root modules are listed WITHOUT a .py suffix. coverage's `source` takes packages and directories and silently ignores anything else - the first version of this change wrote "api_provider.py" and measured exactly nothing new, the statement count sitting at 18,763 while the config looked correct. Caught by comparing the count, not by reading the file. Both lists are hand-maintained in the same file, which is the shape this repo keeps getting bitten by, so tests/test_coverage_scope.py compares them in both directions: every shipped thing is measured, nothing measured is unshipped or nonexistent. Separately, the 300-line register* cap: four functions sit within 17 lines of it (294, 290, 285, 283). A cap everything sits just under binds on whoever next touches one of those functions, for a reason unrelated to their change - the same erosion check-bundle-size.mjs kept suffering. The gate now fails on a FIFTH function entering that band rather than on the four already there, since splitting a 294-line registration function is work to schedule, not to force. Lower the recorded count as they are split; never raise it. Test plan: - CI coverage command, with the widened source: 21,606 statements, 86%, "Required test coverage of 85.0% reached. Total coverage: 85.53%", 3202 passed. - 4 new coverage-scope tests, 1 new cap-headroom test. - ruff and mypy both clean. Worth stating why that mattered here: the first pass at this replaced ".py" suffixes globally in pyproject and silently corrupted three unrelated sections - mutmut's source_paths, mypy's files list, and ruff's api_provider.py per-file-ignore, which is what surfaced it. All three are back to file paths; only the coverage list uses bare names. Co-Authored-By: Claude Opus 5 --- pyproject.toml | 31 ++++++++- tests/test_coverage_scope.py | 92 ++++++++++++++++++++++++++ tests/test_register_function_length.py | 37 +++++++++++ 3 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 tests/test_coverage_scope.py diff --git a/pyproject.toml b/pyproject.toml index d96902db..e62a77e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -165,7 +165,36 @@ faulthandler_timeout = 60 # ~90.4% measured at landing (coverage 7.15.2) - it exists to catch an # actual coverage collapse, not to gate every incremental dip. [tool.coverage.run] -source = ["backend", "graphlink_plugins"] +# EVERY shipped package plus EVERY shipped loose module - kept in step with +# pyproject's own [tool.setuptools] manifest by +# tests/test_coverage_scope.py, which fails when the two drift. +# +# It used to be just backend + graphlink_plugins. Two of the four shipped +# PACKAGES were outside the floor entirely, and so were all 23 root modules - +# including api_provider.py, the code that talks to every model endpoint. +# Measured at the time of widening: 21,606 statements, 86% covered, so the +# 85% floor holds unchanged. provider_runtime is the weak spot inside it +# (ollama_scan 8%, llama_cpp_scan 10%, gemini_transport 29%) and that is +# precisely the point - those numbers were invisible before, and a +# regression in them now counts against the floor. +# +# The root modules are named WITHOUT a .py suffix. coverage's `source` takes +# packages and directories, and silently ignores anything else - the first +# version of this list wrote "api_provider.py" and measured exactly nothing +# new (statement count stayed at 18,763 instead of rising to 21,606). A +# config that looks right and gates nothing is the failure this list was +# widened to fix, so: names, not paths. +source = [ + "backend", "graphlink_plugins", "provider_runtime", "settings_store", "contracts", + "api_provider", "graphlink_artifact_agent", "graphlink_audio", + "graphlink_chart_data", "graphlink_chart_rendering", "graphlink_chat_agent", + "graphlink_desktop", "graphlink_execution_guard", "graphlink_grid_view_settings", + "graphlink_memory", "graphlink_migrations", "graphlink_model_catalog", + "graphlink_navigation_pins", "graphlink_note_agent", "graphlink_process_env", + "graphlink_prompts", "graphlink_scratch_dirs", "graphlink_secrets", + "graphlink_settings_store", "graphlink_task_config", "graphlink_token_estimator", + "graphlink_version", "graphlink_wire_schema", +] omit = ["*/tests/*", "*/__pycache__/*"] [tool.coverage.report] diff --git a/tests/test_coverage_scope.py b/tests/test_coverage_scope.py new file mode 100644 index 00000000..cb76df60 --- /dev/null +++ b/tests/test_coverage_scope.py @@ -0,0 +1,92 @@ +"""The coverage floor has to measure everything that ships. + +[tool.coverage.run].source used to be `["backend", "graphlink_plugins"]`, while +[tool.setuptools] ships FOUR packages and 23 loose root modules. Two shipped +packages - provider_runtime and settings_store - and every root module, +including api_provider.py (the code that talks to every model endpoint), sat +outside the 85% floor entirely. A file nothing measures can regress to zero +and the gate stays green. + +Both lists are hand-maintained in the same file, which is the shape this +codebase keeps getting bitten by: a set asserted by hand, the other set +growing, and nothing failing. So they are compared here instead. + +Deliberately compares against the SHIPPING manifest rather than against +"every .py in the repo": tools/, mutation_tests/ and the test suites are not +shipped and have no business inside a production coverage floor. +""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +PYPROJECT = REPO_ROOT / "pyproject.toml" + +# Measured but not shipped: contracts/ is build-time codegen (it generates the +# TS types the SPA imports), and it has real tests, so it belongs in the floor +# even though no wheel carries it. +_MEASURED_BUT_NOT_SHIPPED = {"contracts"} + + +def _config() -> dict: + return tomllib.loads(PYPROJECT.read_text(encoding="utf-8")) + + +def test_every_shipped_package_is_inside_the_coverage_floor(): + config = _config() + source = set(config["tool"]["coverage"]["run"]["source"]) + # "backend*" -> "backend" + shipped = { + entry.rstrip("*") + for entry in config["tool"]["setuptools"]["packages"]["find"]["include"] + } + missing = sorted(shipped - source) + assert not missing, ( + f"shipped packages outside [tool.coverage.run].source: {missing}. " + "A package nothing measures can regress to zero with the gate still green." + ) + + +def test_every_shipped_root_module_is_inside_the_coverage_floor(): + config = _config() + source = set(config["tool"]["coverage"]["run"]["source"]) + shipped = set(config["tool"]["setuptools"]["py-modules"]) + missing = sorted(shipped - source) + assert not missing, ( + f"shipped root modules outside [tool.coverage.run].source: {missing}" + ) + + +def test_the_coverage_source_lists_nothing_that_does_not_exist(): + """The other direction: a renamed or deleted module left in the list + silently measures nothing, which reads as coverage it does not have. + + Accepts both forms because coverage's `source` names a package/directory + OR a top-level module: `backend` is a directory, `api_provider` resolves + to api_provider.py.""" + config = _config() + stale = [ + entry for entry in config["tool"]["coverage"]["run"]["source"] + if not (REPO_ROOT / entry).exists() and not (REPO_ROOT / f"{entry}.py").exists() + ] + assert not stale, f"[tool.coverage.run].source names things that do not exist: {stale}" + + +def test_the_source_list_adds_nothing_beyond_what_ships(): + """Guards against quietly padding the floor with well-covered code that + is not part of the product.""" + config = _config() + source = set(config["tool"]["coverage"]["run"]["source"]) + shipped = { + entry.rstrip("*") + for entry in config["tool"]["setuptools"]["packages"]["find"]["include"] + } + shipped |= set(config["tool"]["setuptools"]["py-modules"]) + shipped |= _MEASURED_BUT_NOT_SHIPPED + unexpected = sorted(source - shipped) + assert not unexpected, ( + f"[tool.coverage.run].source measures things that do not ship: {unexpected}. " + "If that is deliberate, add it to _MEASURED_BUT_NOT_SHIPPED with a reason." + ) diff --git a/tests/test_register_function_length.py b/tests/test_register_function_length.py index 631ab23a..29d86ccb 100644 --- a/tests/test_register_function_length.py +++ b/tests/test_register_function_length.py @@ -58,6 +58,43 @@ def test_no_register_function_exceeds_the_300_line_cap(): ) +# Below this much headroom, report it. A cap that everything sits just under +# is a cap that will bind on whoever happens to touch one of those functions +# next, for a reason that has nothing to do with their change - the same +# erosion web_ui/scripts/check-bundle-size.mjs kept suffering, where a ratchet +# with 323 bytes left "passed" right up until an unrelated two-line bug fix +# tripped it. 10% of the cap (30 lines) is enough warning to split +# deliberately rather than under duress. +HEADROOM_WARN_LINES = MAX_REGISTER_FUNCTION_LINES // 10 + + +def test_the_number_of_register_functions_near_the_cap_is_not_growing(): + """A cap everything sits just under will bind on whoever happens to touch + one of those functions next, for a reason that has nothing to do with + their change. That is the erosion web_ui/scripts/check-bundle-size.mjs + kept suffering - a ratchet with 323 bytes left "passed" right up until an + unrelated two-line bug fix tripped it. + + Four functions were within 30 lines of the cap when this was added (294, + 290, 285, 283), and splitting a 294-line registration function is real + work that should be scheduled rather than forced. So this does not fail + on those four - it fails on a FIFTH, which is the signal that the + pressure is growing rather than being paid down. + + Lower the recorded count as they are split. Never raise it.""" + tight = sorted( + (MAX_REGISTER_FUNCTION_LINES - length, f"{path.relative_to(REPO_ROOT).as_posix()}::{node.name}", length) + for path, node, length in _register_functions() + if MAX_REGISTER_FUNCTION_LINES - length < HEADROOM_WARN_LINES + ) + assert len(tight) <= 4, ( + f"{len(tight)} register* functions are now within {HEADROOM_WARN_LINES} lines of the " + f"{MAX_REGISTER_FUNCTION_LINES}-line cap, up from the 4 recorded here. Split one " + "before adding another:\n " + + "\n ".join(f"{length} lines ({headroom} left) {name}" for headroom, name, length in tight) + ) + + def test_at_least_one_register_function_is_found(): # A collection bug (wrong glob, wrong name-matching predicate) would # make the test above vacuously pass with zero offenders - this