Improve Fleet-Ops server test coverage

.github/workflows/server.yml

@@ -52,3 +52,12 @@ jobs:
         name: fleetops-coverage-clover
         path: coverage/clover.xml
         if-no-files-found: error
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: coverage/clover.xml
+        disable_search: true
+        flags: backend
+        fail_ci_if_error: false

README.md

@@ -13,6 +13,9 @@
     <a href="https://github.com/fleetbase/fleetops/blob/master/LICENSE.md">
         <img src="https://img.shields.io/badge/license-AGPL--3.0--or--later-blue?style=flat-square" alt="License: AGPL-3.0-or-later">
     </a>
+    <a href="https://codecov.io/gh/fleetbase/fleetops">
+        <img src="https://codecov.io/gh/fleetbase/fleetops/branch/main/graph/badge.svg" alt="Coverage">
+    </a>
     <a href="https://www.npmjs.com/package/@fleetbase/fleetops-engine">
         <img src="https://img.shields.io/badge/npm-%40fleetbase%2Ffleetops--engine-CB3837?style=flat-square" alt="NPM package">
     </a>

composer.json

@@ -22,7 +22,7 @@
     ],
     "require": {
         "php": "^8.0",
-        "barryvdh/laravel-dompdf": "^2.0",
+        "barryvdh/laravel-dompdf": "^3.1",
         "brick/geo": "0.7.2",
         "cknow/laravel-money": "^7.1",
         "fleetbase/core-api": "*",
@@ -90,11 +90,15 @@
             "@test:coverage:clover",
             "@coverage:summary"
         ],
+        "coverage:check": [
+            "@test:coverage:clover",
+            "php scripts/coverage-summary.php coverage/clover.xml --fail-under=100"
+        ],
         "coverage:summary": "php scripts/coverage-summary.php coverage/clover.xml",
         "lint": "php-cs-fixer fix -v",
         "test:lint": "php-cs-fixer fix -v --dry-run",
         "test:coverage": "php scripts/coverage-runner.php --coverage --coverage-text --colors=always",
-        "test:coverage:clover": "mkdir -p coverage && php scripts/coverage-runner.php --coverage-clover=coverage/clover.xml --colors=always",
+        "test:coverage:clover": "mkdir -p coverage && php scripts/coverage-file-runner.php --coverage-clover=coverage/clover.xml --colors=always",
         "test:types": "phpstan analyse --ansi --memory-limit=4G",
         "test:unit": "php scripts/pest-file-runner.php --colors=always",
         "test": [

scripts/coverage-file-runner.php

@@ -0,0 +1,199 @@
+<?php
+
+declare(strict_types=1);
+
+use SebastianBergmann\CodeCoverage\CodeCoverage;
+use SebastianBergmann\CodeCoverage\Report\Clover;
+
+// Merging hundreds of per-file coverage snapshots into one clover report
+// needs more than the default 128M limit
+ini_set('memory_limit', '-1');
+
+$autoloadCandidates = [
+    getcwd() . '/server_vendor/autoload.php',
+    getcwd() . '/vendor/autoload.php',
+];
+
+$autoload = null;
+foreach ($autoloadCandidates as $candidate) {
+    if (is_file($candidate)) {
+        $autoload = $candidate;
+        break;
+    }
+}
+
+if ($autoload === null) {
+    fwrite(STDERR, "Unable to find Composer autoload. Run composer install first.\n");
+    exit(1);
+}
+
+require_once $autoload;
+
+$pestRunner = getcwd() . '/scripts/pest-runner.php';
+if (!is_file($pestRunner)) {
+    fwrite(STDERR, "Unable to find Pest runner at scripts/pest-runner.php.\n");
+    exit(1);
+}
+
+$hasCoverageExtension = extension_loaded('xdebug') || extension_loaded('pcov');
+if (!$hasCoverageExtension) {
+    fwrite(STDERR, "No PHP coverage driver is available.\n\n");
+    fwrite(STDERR, "Install or enable one of:\n");
+    fwrite(STDERR, "  - Xdebug with XDEBUG_MODE=coverage\n");
+    fwrite(STDERR, "  - PCOV\n");
+    fwrite(STDERR, "\n");
+    fwrite(STDERR, 'Current PHP binary: ' . PHP_BINARY . "\n");
+    exit(1);
+}
+
+putenv('XDEBUG_MODE=coverage');
+$_ENV['XDEBUG_MODE']    = 'coverage';
+$_SERVER['XDEBUG_MODE'] = 'coverage';
+
+$args       = array_slice($argv, 1);
+$cloverPath = getcwd() . '/coverage/clover.xml';
+$pestArgs   = [];
+$targets    = [];
+
+foreach ($args as $arg) {
+    if (str_starts_with($arg, '--coverage-clover=')) {
+        $path       = substr($arg, strlen('--coverage-clover='));
+        $cloverPath = str_starts_with($path, '/') ? $path : getcwd() . '/' . $path;
+        continue;
+    }
+
+    if ($arg === '--coverage-clover') {
+        fwrite(STDERR, "Use --coverage-clover=path with coverage-file-runner.php.\n");
+        exit(1);
+    }
+
+    if ($arg !== '' && $arg[0] !== '-') {
+        $targets[] = $arg;
+        continue;
+    }
+
+    if (!str_starts_with($arg, '--coverage-')) {
+        $pestArgs[] = $arg;
+    }
+}
+
+if ($targets === []) {
+    $targets[] = 'server/tests';
+}
+
+$files = [];
+foreach ($targets as $target) {
+    $path = getcwd() . '/' . ltrim($target, '/');
+    if (is_file($path)) {
+        $files[] = $path;
+        continue;
+    }
+
+    if (is_dir($path)) {
+        $iterator = new RecursiveIteratorIterator(
+            new RecursiveDirectoryIterator($path, FilesystemIterator::SKIP_DOTS)
+        );
+
+        foreach ($iterator as $fileInfo) {
+            if ($fileInfo->isFile() && $fileInfo->getExtension() === 'php') {
+                $files[] = $fileInfo->getPathname();
+            }
+        }
+    }
+}
+
+$files = array_values(array_unique($files));
+sort($files);
+
+if ($files === []) {
+    fwrite(STDERR, "Unable to find Pest test files for coverage.\n");
+    exit(1);
+}
+
+$coverageDir = dirname($cloverPath);
+if (!is_dir($coverageDir) && !mkdir($coverageDir, 0777, true) && !is_dir($coverageDir)) {
+    fwrite(STDERR, "Unable to create coverage directory: {$coverageDir}\n");
+    exit(1);
+}
+
+$tmpDir = $coverageDir . '/.coverage-file-runner';
+if (is_dir($tmpDir)) {
+    $iterator = new RecursiveIteratorIterator(
+        new RecursiveDirectoryIterator($tmpDir, FilesystemIterator::SKIP_DOTS),
+        RecursiveIteratorIterator::CHILD_FIRST
+    );
+
+    foreach ($iterator as $fileInfo) {
+        $fileInfo->isDir() ? rmdir($fileInfo->getPathname()) : unlink($fileInfo->getPathname());
+    }
+} elseif (!mkdir($tmpDir, 0777, true) && !is_dir($tmpDir)) {
+    fwrite(STDERR, "Unable to create temporary coverage directory: {$tmpDir}\n");
+    exit(1);
+}
+
+$timeout       = (float) (getenv('PEST_FILE_TIMEOUT') ?: 120);
+$timeoutBinary = trim((string) shell_exec('command -v timeout'));
+$memoryLimit   = getenv('FLEETOPS_COVERAGE_MEMORY_LIMIT') ?: '-1';
+$coverageFiles = [];
+
+foreach ($files as $index => $file) {
+    $relativeFile = str_replace(getcwd() . '/', '', $file);
+    $coverageFile = $tmpDir . '/' . str_pad((string) $index, 4, '0', STR_PAD_LEFT) . '.cov';
+
+    fwrite(STDOUT, "::group::{$relativeFile} coverage\n");
+
+    $command = array_merge([
+        PHP_BINARY,
+        '-d',
+        'memory_limit=' . $memoryLimit,
+        $pestRunner,
+    ], $pestArgs, [
+        '--coverage-php=' . $coverageFile,
+        $file,
+    ]);
+
+    if ($timeoutBinary !== '') {
+        $command = array_merge([$timeoutBinary, "{$timeout}s"], $command);
+    }
+
+    fwrite(STDOUT, '$ ' . implode(' ', array_map('escapeshellarg', $command)) . "\n");
+    passthru(implode(' ', array_map('escapeshellarg', $command)), $exitCode);
+    fwrite(STDOUT, "::endgroup::\n");
+
+    if ($exitCode === 124) {
+        fwrite(STDERR, "\nTimed out after {$timeout} seconds while running {$relativeFile}.\n");
+        exit(1);
+    }
+
+    if ($exitCode !== 0) {
+        fwrite(STDERR, "\nPest coverage failed for {$relativeFile} with exit code {$exitCode}.\n");
+        exit($exitCode);
+    }
+
+    $coverageFiles[] = $coverageFile;
+}
+
+$merged = null;
+foreach ($coverageFiles as $coverageFile) {
+    $coverage = include $coverageFile;
+
+    if (!$coverage instanceof CodeCoverage) {
+        fwrite(STDERR, "Invalid coverage artifact: {$coverageFile}\n");
+        exit(1);
+    }
+
+    if ($merged === null) {
+        $merged = $coverage;
+        continue;
+    }
+
+    $merged->merge($coverage);
+}
+
+if (!$merged instanceof CodeCoverage) {
+    fwrite(STDERR, "No coverage data was produced.\n");
+    exit(1);
+}
+
+(new Clover())->process($merged, $cloverPath);
+fwrite(STDOUT, "Wrote Clover coverage to {$cloverPath}\n");

scripts/coverage-runner.php

@@ -43,7 +43,8 @@
     $args[] = 'server/tests';
 }
 
-$command        = array_merge([PHP_BINARY, $pestRunner], $args);
+$memoryLimit = getenv('FLEETOPS_COVERAGE_MEMORY_LIMIT') ?: '-1';
+$command     = array_merge([PHP_BINARY, '-d', 'memory_limit=' . $memoryLimit, $pestRunner], $args);
 $escapedCommand = implode(' ', array_map('escapeshellarg', $command));
 
 passthru($escapedCommand, $exitCode);

scripts/coverage-summary.php

@@ -2,7 +2,20 @@
 
 declare(strict_types=1);
 
-$cloverPath = $argv[1] ?? 'coverage/clover.xml';
+$cloverPath = 'coverage/clover.xml';
+$failUnder  = null;
+
+foreach (array_slice($argv, 1) as $arg) {
+    if (str_starts_with($arg, '--fail-under=')) {
+        $failUnder = (float) substr($arg, strlen('--fail-under='));
+
+        continue;
+    }
+
+    if ($arg !== '') {
+        $cloverPath = $arg;
+    }
+}
 
 if (!is_file($cloverPath)) {
     fwrite(STDERR, "Coverage file not found: {$cloverPath}\n");
@@ -26,6 +39,27 @@ function intMetric(SimpleXMLElement $node, string $name): int
     return (int) ($node->metrics[$name] ?? 0);
 }
 
+function hasMetric(SimpleXMLElement $node, string $name): bool
+{
+    return isset($node->metrics[$name]);
+}
+
+function deriveCoveredClasses(SimpleXMLElement $project): int
+{
+    $coveredClasses = 0;
+
+    foreach ($project->xpath('.//class') ?: [] as $class) {
+        $methods        = intMetric($class, 'methods');
+        $coveredMethods = intMetric($class, 'coveredmethods');
+
+        if ($methods > 0 && $coveredMethods >= $methods) {
+            $coveredClasses++;
+        }
+    }
+
+    return $coveredClasses;
+}
+
 $project = $xml->project;
 $metrics = $project->metrics;
 
@@ -34,7 +68,7 @@ function intMetric(SimpleXMLElement $node, string $name): int
 $methods           = (int) ($metrics['methods'] ?? 0);
 $coveredMethods    = (int) ($metrics['coveredmethods'] ?? 0);
 $classes           = (int) ($metrics['classes'] ?? 0);
-$coveredClasses    = (int) ($metrics['coveredclasses'] ?? 0);
+$coveredClasses    = hasMetric($project, 'coveredclasses') ? (int) $metrics['coveredclasses'] : deriveCoveredClasses($project);
 
 $files       = [];
 $directories = [];
@@ -108,3 +142,8 @@ function intMetric(SimpleXMLElement $node, string $name): int
     $relativePath = preg_replace('#^' . preg_quote(getcwd(), '#') . '/?#', '', $file['path']);
     printf("  %6.2f%%  %5d/%-5d  %s\n", $file['percent'], $file['covered'], $file['statements'], $relativePath ?: $file['path']);
 }
+
+if ($failUnder !== null && coveragePercent($coveredStatements, $statements) < $failUnder) {
+    fwrite(STDERR, sprintf("\nCoverage %.2f%% is below the required %.2f%% line threshold.\n", coveragePercent($coveredStatements, $statements), $failUnder));
+    exit(1);
+}