Detect a node that dies when nothing else is left to report it

.github/workflows/ci.yml

@@ -108,8 +108,11 @@ jobs:
           RMW_IMPLEMENTATION: ${{ matrix.ros_distro == 'humble' && 'rmw_cyclonedds_cpp' || '' }}
         run: |
           source /opt/ros/${{ matrix.ros_distro }}/setup.bash
+          # ros2_medkit_graph_watchdog is tested in graph-watchdog below. Its end-to-end suite
+          # is 24 minutes on its own, and this step is capped rather than the job, so a package
+          # that overruns takes every package behind it down and reports no test name doing it.
           colcon test --return-code-on-test-failure \
-            --packages-skip ros2_medkit_opcua \
+            --packages-skip ros2_medkit_opcua ros2_medkit_graph_watchdog \
             --ctest-args -LE linter \
             --event-handlers console_direct+
 
@@ -126,6 +129,120 @@ jobs:
             log/
             build/*/test_results/
 
+  # ros2_medkit_graph_watchdog on the two distros build-and-test covers, in a job of its own.
+  # Same reasoning as sanitizer-graph-watchdog in quality.yml: the package's end-to-end suite runs
+  # 24 minutes, the sweep it left had 3 minutes of headroom on humble and none at all on lyrical,
+  # and a suite that grows with every new detector should not decide whether the packages behind it
+  # get to run. Not path-filtered: the plugin drives the gateway, the fault manager and discovery,
+  # so the changes most likely to break it are outside its own tree.
+  graph-watchdog:
+    name: graph_watchdog (${{ matrix.ros_distro }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - ros_distro: humble
+            os_image: ubuntu:jammy
+          - ros_distro: lyrical
+            os_image: ubuntu:resolute
+    container:
+      image: ${{ matrix.os_image }}
+    timeout-minutes: 90
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Install Git
+        run: |
+          apt-get update
+          apt-get install -y git
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Pre-install ROS 2 apt source
+        uses: ./.github/actions/ros-apt-source
+
+      - name: Set up ROS 2 ${{ matrix.ros_distro }}
+        uses: ros-tooling/setup-ros@v0.7
+        with:
+          required-ros-distributions: ${{ matrix.ros_distro }}
+
+      - name: Install ccache
+        run: apt-get install -y ccache
+
+      - name: Restore ccache
+        # Restore, never save. This job compiles a subset of the sources build-and-test compiles,
+        # with the same flags, so that job's cache is a superset of what this one needs and serves
+        # its misses. A second copy would spend the repository's Actions cache quota twice over for
+        # one set of objects.
+        uses: actions/cache/restore@v4
+        with:
+          path: /root/.cache/ccache
+          key: ccache-${{ matrix.ros_distro }}-${{ github.sha }}
+          restore-keys: |
+            ccache-${{ matrix.ros_distro }}-
+
+      - name: Install dependencies
+        run: |
+          apt-get update
+          apt-get install -y ros-${{ matrix.ros_distro }}-test-msgs
+          if [ "${{ matrix.ros_distro }}" = "humble" ]; then
+            apt-get install -y ros-humble-rmw-cyclonedds-cpp
+          fi
+          source /opt/ros/${{ matrix.ros_distro }}/setup.bash
+          rosdep update
+          # Same skip-keys as build-and-test: the linters run only in the Jazzy quality job.
+          rosdep install --from-paths src --ignore-src -y \
+            --skip-keys "ament_cmake_clang_tidy ament_cmake_clang_format"
+
+      - name: Build packages
+        env:
+          CCACHE_DIR: /root/.cache/ccache
+          CCACHE_MAXSIZE: 500M
+          CCACHE_SLOPPINESS: pch_defines,time_macros
+        run: |
+          source /opt/ros/${{ matrix.ros_distro }}/setup.bash
+          ccache -z
+          # --packages-up-to, not the workspace: the chain reaches the gateway, the fault manager
+          # and ros2_medkit_integration_tests, which is a test dependency and is where the launch
+          # helpers and the demo nodes the end-to-end scenarios start actually live. opcua is not in
+          # that chain, so it needs no explicit skip here.
+          colcon build --symlink-install \
+            --packages-up-to ros2_medkit_graph_watchdog \
+            --cmake-args -DCMAKE_BUILD_TYPE=Release \
+            --event-handlers console_direct+
+          ccache -s
+          ./scripts/ccache_report.sh "${{ matrix.ros_distro }}-graph-watchdog"
+
+      - name: Run graph_watchdog tests
+        timeout-minutes: 45
+        env:
+          # The DDS choice belongs to the distro, not to this package - see the long note in
+          # build-and-test for why humble is forced onto CycloneDDS and lyrical is not.
+          RMW_IMPLEMENTATION: ${{ matrix.ros_distro == 'humble' && 'rmw_cyclonedds_cpp' || '' }}
+        run: |
+          source /opt/ros/${{ matrix.ros_distro }}/setup.bash
+          colcon test --return-code-on-test-failure \
+            --packages-select ros2_medkit_graph_watchdog \
+            --ctest-args -LE linter \
+            --event-handlers console_direct+
+
+      - name: Show test results
+        if: always()
+        run: colcon test-result --verbose
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-graph-watchdog-${{ matrix.ros_distro }}
+          path: |
+            log/
+            build/*/test_results/
+
   # Builds AND tests Jazzy. These were two jobs (jazzy-build -> jazzy-test)
   # passing a tarred build/ + install/ tree between them. That split dates from
   # when lint ran off the same artifact in parallel; lint has since moved to

.github/workflows/quality.yml

@@ -382,6 +382,11 @@ jobs:
           failed=0
           for pkg_dir in build/ros2_medkit_*/; do
             pkg=$(basename "$pkg_dir")
+            # graph_watchdog is tested in sanitizer-graph-watchdog. Its
+            # end-to-end suite is 24 minutes under instrumentation, which is
+            # more than the headroom left here, and a package that overruns
+            # this step takes every package after it down with it.
+            if [ "$pkg" = "ros2_medkit_graph_watchdog" ]; then continue; fi
             echo "::group::Testing $pkg"
             (cd "$pkg_dir" && ctest -LE "linter" --output-on-failure) || failed=1
             echo "::endgroup::"
@@ -392,6 +397,7 @@ jobs:
         if: always()
         run: |
           for pkg_dir in build/ros2_medkit_*/; do
+            if [ "$(basename "$pkg_dir")" = "ros2_medkit_graph_watchdog" ]; then continue; fi
             colcon test-result --test-result-base "$pkg_dir" --verbose 2>/dev/null || true
           done
 
@@ -500,6 +506,11 @@ jobs:
           failed=0
           for pkg_dir in build/ros2_medkit_*/; do
             pkg=$(basename "$pkg_dir")
+            # graph_watchdog is tested in sanitizer-graph-watchdog. Its
+            # end-to-end suite is 24 minutes under instrumentation, which is
+            # more than the headroom left here, and a package that overruns
+            # this step takes every package after it down with it.
+            if [ "$pkg" = "ros2_medkit_graph_watchdog" ]; then continue; fi
             echo "::group::Testing $pkg"
             (cd "$pkg_dir" && ctest -j1 -LE "linter" --output-on-failure) || failed=1
             echo "::endgroup::"
@@ -510,5 +521,152 @@ jobs:
         if: always()
         run: |
           for pkg_dir in build/ros2_medkit_*/; do
+            if [ "$(basename "$pkg_dir")" = "ros2_medkit_graph_watchdog" ]; then continue; fi
             colcon test-result --test-result-base "$pkg_dir" --verbose 2>/dev/null || true
           done
+
+  # ros2_medkit_graph_watchdog's end-to-end suite runs 24 minutes under
+  # instrumentation, against a 45-minute test budget the rest of the workspace
+  # already spends 22 to 38 of. Testing it here instead of in the workspace
+  # sweeps gives it a budget of its own, and stops a suite that grows with every
+  # new detector from deciding whether the packages behind it get to run at all.
+  # It is not path-filtered: the plugin drives the gateway, the fault manager and
+  # discovery, so the changes most likely to break it are not in its own tree.
+  sanitizer-graph-watchdog:
+    name: Sanitizer ${{ matrix.name }} (graph_watchdog)
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sanitizer: asan
+            name: ASan + UBSan
+            cmake_sanitizer: asan,ubsan
+            # Both sizes are the ones the matching workspace job argued for.
+            # This job restores that job's cache and must not evict what it
+            # restored, so it cannot be smaller here.
+            ccache_size: 2G
+          - sanitizer: tsan
+            name: TSan
+            cmake_sanitizer: tsan
+            ccache_size: 1.5G
+    container:
+      image: ubuntu:noble
+      # See sanitizer-asan: the instrumented build tree does not fit the
+      # container overlay on small GitHub runners, so build on /mnt.
+      volumes:
+        - "/mnt:/mnt"
+    # A cold cache builds the chain from scratch, which is the ~20 min the TSan
+    # workspace job measures, before the 45-minute test budget starts.
+    timeout-minutes: 90
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Install Git
+        run: |
+          apt-get update
+          apt-get install -y git
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Pre-install ROS 2 apt source
+        uses: ./.github/actions/ros-apt-source
+
+      - name: Set up ROS 2 Jazzy
+        uses: ros-tooling/setup-ros@v0.7
+        with:
+          required-ros-distributions: jazzy
+
+      - name: Install ccache
+        run: apt-get install -y ccache
+
+      - name: Restore ccache
+        # Restore, never save. This job compiles a subset of the sources
+        # sanitizer-${{ matrix.sanitizer }} compiles, with the same flags, so
+        # that job's cache is a superset of what this one needs and serves its
+        # misses. A second copy would buy nothing and would spend the
+        # repository's 10 GB Actions cache quota twice for one set of objects.
+        uses: actions/cache/restore@v4
+        with:
+          path: /root/.cache/ccache
+          key: ccache-jazzy-${{ matrix.sanitizer }}-${{ github.sha }}
+          restore-keys: |
+            ccache-jazzy-${{ matrix.sanitizer }}-
+
+      - name: Install dependencies
+        run: |
+          apt-get update
+          apt-get install -y ros-jazzy-test-msgs
+          source /opt/ros/jazzy/setup.bash
+          rosdep update
+          rosdep install --from-paths src --ignore-src -y
+
+      - name: Redirect heavy build output to /mnt
+        run: |
+          # Same reasoning as the workspace sanitizer jobs, and the same need:
+          # the chain built here still includes the gateway.
+          mkdir -p /mnt/gw/build /mnt/gw/tmp
+          ln -sfn /mnt/gw/build build
+          df -h / /mnt
+
+      - name: Build with ${{ matrix.name }}
+        env:
+          CCACHE_DIR: /root/.cache/ccache
+          CCACHE_MAXSIZE: ${{ matrix.ccache_size }}
+          CCACHE_SLOPPINESS: pch_defines,time_macros
+          TMPDIR: /mnt/gw/tmp
+        run: |
+          source /opt/ros/jazzy/setup.bash
+          ccache -z
+          # --packages-up-to rather than the workspace: the chain reaches the
+          # gateway, the fault manager and ros2_medkit_integration_tests, which
+          # is a test dependency and is where the launch helpers and the demo
+          # nodes the end-to-end scenarios start actually live.
+          colcon build --symlink-install \
+            --packages-up-to ros2_medkit_graph_watchdog \
+            --cmake-args -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+              -DSANITIZER=${{ matrix.cmake_sanitizer }} \
+            --event-handlers console_direct+
+          ccache -s
+          ./scripts/ccache_report.sh jazzy-${{ matrix.sanitizer }}-graph-watchdog
+          df -h / /mnt
+
+      - name: Extend test timeouts for sanitizer overhead
+        run: |
+          # The same generic x3 rewrite the workspace sanitizer jobs apply - see
+          # the comment in sanitizer-asan for why it is not a list of literals.
+          find build/ -name "CTestTestfile.cmake" -exec \
+            perl -pi -e 's/\bTIMEOUT "(\d+)"/sprintf(q{TIMEOUT "%d"}, $1 * 3)/ge' {} +
+          find build/ -name "CTestTestfile.cmake" -exec cat {} + \
+            | grep -oE 'TIMEOUT "[0-9]+"' | sort -t'"' -k2 -n | uniq -c
+
+      - name: Run graph_watchdog tests with ${{ matrix.name }}
+        timeout-minutes: 45
+        env:
+          # Same factor as the ctest TIMEOUT rewrite above, for the wall-clock
+          # budgets tests assert internally - ctest's clock cannot reach those.
+          MEDKIT_TEST_TIME_SCALE: 3
+          # Lets tests size instrumented-only-expensive resources down.
+          MEDKIT_TEST_SANITIZED: 1
+        run: |
+          if [ "${{ matrix.sanitizer }}" = "tsan" ]; then
+            export TSAN_OPTIONS="halt_on_error=0:history_size=4:suppressions=$(pwd)/tsan_suppressions.txt"
+          else
+            # detect_leaks=0: FastDDS allocator leaks on shutdown (not our code)
+            # new_delete_type_mismatch=0: ROS 2 DDS scalar/array new/delete mismatch
+            export ASAN_OPTIONS=halt_on_error=1:detect_leaks=0:new_delete_type_mismatch=0
+            export UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1
+          fi
+          source /opt/ros/jazzy/setup.bash
+          source install/setup.bash
+          cd build/ros2_medkit_graph_watchdog
+          ctest -j1 -LE "linter" --output-on-failure
+
+      - name: Show test results
+        if: always()
+        run: |
+          colcon test-result --test-result-base build/ros2_medkit_graph_watchdog \
+            --verbose 2>/dev/null || true