self-healing: complete phase-2 port, docs, and e2e/ci coverage

Author: j-rafique

.github/workflows/tests.yml

@@ -70,6 +70,44 @@ jobs:
       - name: Run cascade e2e tests
         run: make test-cascade
 
+  self-healing-e2e-tests:
+    name: self-healing-e2e-tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6.0.1
+
+      - name: Setup Go and system deps
+        uses: ./.github/actions/setup-env
+        # with:
+        #   bust_lumera_retag: 'true'
+
+      - name: Go mod tidy
+        run: go mod tidy
+
+      - name: Install Lumera
+        run: make install-lumera
+
+      - name: Setup Supernode environments
+        run: make setup-supernodes
+
+      - name: Run self-healing e2e tests
+        run: make test-self-healing
+
+      - name: Dump self-healing runtime traces
+        if: always()
+        run: |
+          cd tests/system
+          for f in supernode0.out supernode1.out supernode2.out; do
+            echo "==== ${f} (self-healing excerpts) ===="
+            if [ -f "$f" ]; then
+              grep -E "self-healing|sh-e2e-happy|RequestSelfHealing|VerifySelfHealing|CommitSelfHealing|reconstructed_hash_hex|observer" "$f" | tail -n 200 || true
+            else
+              echo "missing ${f}"
+            fi
+          done
+
   # sn-manager-e2e-tests:
   #   name: sn-manager-e2e-tests
   #   runs-on: ubuntu-latest

Makefile

@@ -120,7 +120,7 @@ release:
 ###################################################
 ###              Tests and Simulation           ###
 ###################################################
-.PHONY: test-e2e test-unit test-integration test-system test-cascade test-sn-manager
+.PHONY: test-e2e test-unit test-integration test-system test-cascade test-self-healing test-sn-manager
 .PHONY: install-lumera setup-supernodes system-test-setup install-deps
 .PHONY: gen-cascade gen-supernode
 test-unit:
@@ -152,7 +152,7 @@ gen-supernode:
 		--grpc-gateway_out=gen \
 		--grpc-gateway_opt=paths=source_relative \
 		--openapiv2_out=gen \
-		proto/supernode/service.proto proto/supernode/status.proto proto/supernode/storage_challenge.proto
+		proto/supernode/service.proto proto/supernode/status.proto proto/supernode/storage_challenge.proto proto/supernode/self_healing.proto
 
 # Define the paths
 SUPERNODE_SRC=supernode/main.go
@@ -201,8 +201,12 @@ test-cascade:
 	@echo "Running cascade e2e tests..."
 	@cd tests/system && ${GO} mod tidy && ${GO} test -tags=system_test -v -run TestCascadeE2E .
 
+# Run self-healing e2e tests only
+test-self-healing:
+	@echo "Running self-healing e2e tests..."
+	@cd tests/system && ${GO} mod tidy && ${GO} test -tags=system_test -v -run '^TestSelfHealingE2E' .
+
 # Run sn-manager e2e tests only
 test-sn-manager:
 	@echo "Running sn-manager e2e tests..."
 	@cd tests/system && ${GO} test -tags=system_test -v -run '^TestSNManager' .
-