feat(pd): add quorum-aware /v1/ready endpoint and raft gauges

docker/README.md

@@ -202,6 +202,24 @@ done
 curl -fsS http://localhost:8088/about
 ```
 
+PD answers two unauthenticated probe endpoints. `/v1/health` is liveness only:
+it returns `200` as soon as the REST listener is up, even when the PD has no
+raft leader. `/v1/ready` returns `200` only while the PD sees a raft leader,
+and `503` otherwise. A single PD elects itself; three PDs become ready once
+two of them can talk to each other.
+
+The healthchecks in these files still gate on `/v1/health`, because
+`/v1/ready` ships from the next release onwards while the files run published
+images. Two things to know before pointing them at readiness:
+
+- Match on the body, not the status code. PD answers `200` with
+  `{"status":-1,"error":"Unauthorized!"}` on every path its auth interceptor
+  does not exclude, a path that does not exist included, so a status-only
+  probe reads a PD too old to have `/v1/ready` as ready. Gate with
+  `curl -fsS http://localhost:8620/v1/ready | grep -q '"ready":true'` instead.
+- Pin `HUGEGRAPH_VERSION` to a release that carries the endpoint, or build the
+  images from source with `docker-compose.dev.yml`.
+
 Open `http://localhost:8088` and sign in as `admin` with the password from
 `.env`.
 

hugegraph-pd/README.md

@@ -284,6 +284,8 @@ docker/docker-compose-3pd-3store-3server.yml
 
 PD exposes metrics via REST API at:
 - Health check: `http://<pd-host>:8620/actuator/health`
+- Liveness: `http://<pd-host>:8620/v1/health` (REST listener is up)
+- Readiness: `http://<pd-host>:8620/v1/ready` (`200` only while the PD sees a raft leader)
 - Metrics: `http://<pd-host>:8620/actuator/metrics`
 
 ## Community

hugegraph-pd/docs/api-reference.md

@@ -774,6 +774,46 @@ curl http://localhost:8620/actuator/health
 }
 ```
 
+### Liveness and Readiness
+
+Two unauthenticated endpoints are meant for probes and startup gates:
+
+| Endpoint | Meaning | Status |
+|----------|---------|--------|
+| `GET /v1/health` | Liveness: the REST listener is up. Does not consult raft. | always `200` |
+| `GET /v1/ready` | Readiness: the raft node is active and sees a leader, so this PD is inside a quorum. | `200` when ready, `503` otherwise |
+
+```bash
+curl -i http://localhost:8620/v1/ready
+```
+
+**Response** (leader of a healthy cluster):
+```json
+{
+  "ready": true,
+  "state": "STATE_LEADER",
+  "isLeader": true
+}
+```
+
+A follower reports `"state": "STATE_FOLLOWER"` with `"isLeader": false`. When
+the quorum is lost the PD keeps answering `/v1/health` with `200` but
+`/v1/ready` turns into `503` with `"ready": false`. Being unauthenticated, the
+body carries no cluster addresses; the leader's address stays on `/v1/members`.
+
+Point Kubernetes readiness probes, `depends_on` healthchecks and any
+"wait for PD" script at `/v1/ready`; keep liveness probes on `/v1/health`
+so a PD that merely lost its leader is not restarted.
+
+Match on the body rather than on the status code alone. PD's auth interceptor
+rejects a request it does not exclude by writing an error envelope without
+setting a status, so any unknown path answers `200` with
+`{"status":-1,"error":"Unauthorized!"}`. A status-only probe therefore reads a
+PD older than this endpoint as ready. A shell gate should use
+`curl -fsS http://<pd-host>:8620/v1/ready | grep -q '"ready":true'`, and a
+Kubernetes `httpGet` probe should be paired with a PD image that carries the
+endpoint.
+
 ### Metrics
 
 ```bash
@@ -796,6 +836,21 @@ pd_store_count{state="Offline"} 0.0
 pd_partition_count 36.0
 ```
 
+#### Raft membership gauges
+
+Exported on `/actuator/prometheus` for alerting on quorum loss:
+
+| Gauge | Value |
+|-------|-------|
+| `hg_raft_leader` | `1` on the raft leader, `0` elsewhere |
+| `hg_raft_has_leader` | `1` while this PD sees a leader (is inside a quorum), `0` otherwise |
+| `hg_raft_alive_peers` | On the leader, the number of peers (itself included) heard from within the leader lease timeout (90% of the election timeout by default); `NaN` on other nodes |
+
+A cluster has lost its quorum when `sum(hg_raft_leader) == 0` or when
+`hg_raft_has_leader == 0` on every member. Both are briefly true during a
+normal election, so alert on them with a `for:` clause longer than the
+election timeout rather than on the instantaneous value.
+
 ### Partition API
 
 #### List Partitions

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftEngine.java

@@ -46,6 +46,7 @@
 import com.alipay.sofa.jraft.Status;
 import com.alipay.sofa.jraft.conf.Configuration;
 import com.alipay.sofa.jraft.core.Replicator;
+import com.alipay.sofa.jraft.core.State;
 import com.alipay.sofa.jraft.entity.PeerId;
 import com.alipay.sofa.jraft.entity.Task;
 import com.alipay.sofa.jraft.error.RaftError;
@@ -69,7 +70,7 @@ public class RaftEngine {
     private PDConfig.Raft config;
     private RaftGroupService raftGroupService;
     private RpcServer rpcServer;
-    private Node raftNode;
+    private volatile Node raftNode;
     private RaftRpcClient raftRpcClient;
 
     public RaftEngine() {
@@ -203,7 +204,105 @@ public void shutDown() {
     }
 
     public boolean isLeader() {
-        return this.raftNode.isLeader(true);
+        Node node = this.raftNode;
+        return node != null && node.isLeader(true);
+    }
+
+    /**
+     * Whether this node currently knows a raft leader.
+     * <p>
+     * A follower only keeps its leader id while heartbeats keep arriving inside the election
+     * timeout, and a leader only keeps its role while it can reach a quorum. A non-null leader
+     * therefore means this node is part of a quorum from its own point of view, which is the
+     * signal a readiness probe needs.
+     */
+    public boolean hasLeader() {
+        return hasLeader(this.raftNode);
+    }
+
+    private static boolean hasLeader(Node node) {
+        if (node == null) {
+            return false;
+        }
+        PeerId leader = node.getLeaderId();
+        return leader != null && !leader.isEmpty();
+    }
+
+    /**
+     * Whether this node can take part in serving requests: the raft node has been started,
+     * is in an active state (leader, follower or transferring leadership) and sees a leader.
+     * Unlike a plain liveness check this turns false as soon as the quorum is lost.
+     */
+    public boolean isReady() {
+        return getRaftStatus().isReady();
+    }
+
+    /**
+     * Take a consistent view of the local raft state. Every field is derived from one
+     * {@link Node} reference and a single {@code getLeaderId()} read, so a step-down while
+     * the view is being built cannot report a ready node that knows no leader.
+     */
+    public RaftStatus getRaftStatus() {
+        Node node = this.raftNode;
+        if (node == null) {
+            return new RaftStatus(false, State.STATE_UNINITIALIZED.name(), false);
+        }
+        State state = node.getNodeState();
+        boolean active = state != null && state.isActive();
+        return new RaftStatus(active && hasLeader(node),
+                              state == null ? State.STATE_UNINITIALIZED.name() : state.name(),
+                              node.isLeader(true));
+    }
+
+    /**
+     * Immutable view of the raft state behind {@code GET /v1/ready}. It carries no cluster
+     * addresses: the endpoint is unauthenticated, and the leader's address stays on the
+     * authenticated {@code /v1/members}.
+     */
+    public static final class RaftStatus {
+
+        private final boolean ready;
+        private final String state;
+        private final boolean localLeader;
+
+        RaftStatus(boolean ready, String state, boolean localLeader) {
+            this.ready = ready;
+            this.state = state;
+            this.localLeader = localLeader;
+        }
+
+        public boolean isReady() {
+            return this.ready;
+        }
+
+        /**
+         * @return the jraft node state name, never null
+         */
+        public String getState() {
+            return this.state;
+        }
+
+        public boolean isLocalLeader() {
+            return this.localLeader;
+        }
+    }
+
+    /**
+     * Number of raft peers, this node included, that the leader has heard from within the
+     * leader lease timeout, which jraft derives as 90% of the election timeout by default.
+     * Only the leader tracks replication state, so any other node returns -1.
+     */
+    public int getAlivePeerCount() {
+        Node node = this.raftNode;
+        if (node == null || !node.isLeader(true)) {
+            return -1;
+        }
+        try {
+            return node.listAlivePeers().size();
+        } catch (IllegalStateException e) {
+            // Lost leadership between the check and the call
+            return -1;
+        }
     }
 
     /**
@@ -232,7 +331,8 @@ public PDConfig.Raft getConfig() {
     }
 
     public PeerId getLeader() {
-        return raftNode.getLeaderId();
+        Node node = this.raftNode;
+        return node == null ? null : node.getLeaderId();
     }
 
     /**

hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/metrics/PDMetrics.java

@@ -33,6 +33,7 @@
 import org.apache.hugegraph.pd.grpc.Metapb;
 import org.apache.hugegraph.pd.grpc.Metapb.ShardGroup;
 import org.apache.hugegraph.pd.model.GraphStatistics;
+import org.apache.hugegraph.pd.raft.RaftEngine;
 import org.apache.hugegraph.pd.service.PDRestService;
 import org.apache.hugegraph.pd.service.PDService;
 
@@ -76,7 +77,28 @@ private void registerMeters() {
         Gauge.builder(PREFIX + ".terms", () -> setTerms())
              .description("term of partitions in PD")
              .register(registry);
+        registerRaftMeters();
+    }
 
+    /**
+     * Raft membership gauges so operators can alert on quorum loss. They mirror what
+     * {@code GET /v1/ready} answers: a PD that sees no leader is outside a quorum.
+     */
+    private void registerRaftMeters() {
+        RaftEngine raft = RaftEngine.getInstance();
+        Gauge.builder(PREFIX + ".raft.leader", () -> raft.isLeader() ? 1 : 0)
+             .description("1 if this PD is the raft leader, 0 otherwise")
+             .register(registry);
+        Gauge.builder(PREFIX + ".raft.has_leader", () -> raft.hasLeader() ? 1 : 0)
+             .description("1 if this PD sees a raft leader, i.e. is part of a quorum, 0 otherwise")
+             .register(registry);
+        Gauge.builder(PREFIX + ".raft.alive_peers", () -> {
+                 int alive = raft.getAlivePeerCount();
+                 return alive < 0 ? Double.NaN : alive;
+             })
+             .description("Number of raft peers, itself included, the leader has heard from " +
+                          "within the leader lease timeout; NaN on non-leader nodes")
+             .register(registry);
     }
 
     private long updateGraphs() {

hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/StoreAPI.java

@@ -22,6 +22,7 @@
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -31,11 +32,14 @@
 import org.apache.hugegraph.pd.model.RestApiResponse;
 import org.apache.hugegraph.pd.model.StoreRestRequest;
 import org.apache.hugegraph.pd.model.TimeRangeRequest;
+import org.apache.hugegraph.pd.raft.RaftEngine;
 import org.apache.hugegraph.pd.service.PDRestService;
 import org.apache.hugegraph.pd.util.DateUtil;
 import org.apache.hugegraph.pd.util.StoreRestAddressUtil;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -378,6 +382,10 @@ class StoreStatistics {
      * Check Service Health Status
      * This interface is used to check the health status of the service by accessing the /health
      * path via a GET request.
+     * <p>
+     * This is a liveness signal only: it answers 200 as soon as the REST listener is up and
+     * does not consult the raft state. Use {@link #checkReady()} to find out whether this PD
+     * can actually serve.
      *
      * @return Returns a string indicating the service's health status. Typically, an empty
      * string indicates the service is healthy.
@@ -386,4 +394,27 @@ class StoreStatistics {
     public Serializable checkHealthy() {
         return "";
     }
+
+    /**
+     * Check Service Readiness
+     * Answers 200 only when this PD is part of a raft quorum, that is, the raft node is active
+     * and knows the current leader. Otherwise answers 503 so that anything gating on PD
+     * (the compose healthcheck in front of Stores, a Kubernetes readiness probe)
+     * is held back until the PD can serve. Like /health this endpoint needs no authentication.
+     *
+     * @return JSON with the readiness flag, the local raft state and whether this node is the
+     * leader. It carries no cluster addresses, since the endpoint is unauthenticated.
+     */
+    @GetMapping(value = "/ready", produces = MediaType.APPLICATION_JSON_VALUE)
+    public ResponseEntity<Map<String, Object>> checkReady() {
+        RaftEngine.RaftStatus status = RaftEngine.getInstance().getRaftStatus();
+
+        Map<String, Object> body = new LinkedHashMap<>();
+        body.put("ready", status.isReady());
+        body.put("state", status.getState());
+        body.put("isLeader", status.isLocalLeader());
+        HttpStatus httpStatus = status.isReady() ? HttpStatus.OK
+                                                 : HttpStatus.SERVICE_UNAVAILABLE;
+        return ResponseEntity.status(httpStatus).body(body);
+    }
 }

hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/interceptor/AuthenticationConfigurer.java

@@ -32,6 +32,7 @@ public class AuthenticationConfigurer implements WebMvcConfigurer {
     public void addInterceptors(InterceptorRegistry registry) {
         registry.addInterceptor(restAuthentication)
                 .addPathPatterns("/**")
-                .excludePathPatterns("/actuator/*", "/v1/health", "/v1/prom/targets/*");
+                .excludePathPatterns("/actuator/*", "/v1/health", "/v1/ready",
+                                     "/v1/prom/targets/*");
     }
 }

hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreSuiteTest.java

@@ -22,6 +22,7 @@
 import org.apache.hugegraph.pd.raft.IpAuthHandlerTest;
 import org.apache.hugegraph.pd.raft.RaftEngineIpAuthIntegrationTest;
 import org.apache.hugegraph.pd.raft.RaftEngineLeaderAddressTest;
+import org.apache.hugegraph.pd.raft.RaftEngineReadinessTest;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
 
@@ -43,6 +44,7 @@
         IpAuthHandlerTest.class,
         RaftEngineIpAuthIntegrationTest.class,
         RaftEngineLeaderAddressTest.class,
+        RaftEngineReadinessTest.class,
         // StoreNodeServiceTest.class,
 })
 @Slf4j