diff --git a/docker/README.md b/docker/README.md index 0bb74cf81f..5adff0f72f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -202,6 +202,30 @@ done curl -fsS http://localhost:8088/about ``` +PD answers two unauthenticated probe endpoints. `/v1/health` is liveness only: +it returns `200` as soon as the REST listener is up, even when the PD has no +raft leader. `/v1/ready` returns `200` only while the PD sees a raft leader, +and `503` otherwise. A single PD elects itself; three PDs become ready once +two of them can talk to each other. + +The healthchecks in these files still gate on `/v1/health`, because +`/v1/ready` ships from the next release onwards while the files run published +images. Two things to know before pointing them at readiness: + +- Match on the body, not the status code. As of 1.7.0 PD answers `200` with + `{"status":-1,"error":"Unauthorized!"}` on every path its auth interceptor + does not exclude, a path that does not exist included, so a status-only + probe reads a PD too old to have `/v1/ready` as ready. The body match holds + whichever status a refusal carries. Gate with + `curl -fsS http://localhost:8620/v1/ready | grep -q '"ready":true'` instead. +- Pin `HUGEGRAPH_VERSION` to a release that carries the endpoint, or build the + images from source with `docker-compose.dev.yml`. + +The `HEALTHCHECK` baked into `hugegraph-pd/Dockerfile` is `/v1/health` as well. +Both compose files override it, so it governs `docker run` and anything else +inheriting the image probe, and those keep reading a PD without a quorum as +healthy. + Open `http://localhost:8088` and sign in as `admin` with the password from `.env`. diff --git a/hugegraph-pd/README.md b/hugegraph-pd/README.md index 794dba9b98..c940dae4c9 100644 --- a/hugegraph-pd/README.md +++ b/hugegraph-pd/README.md @@ -284,6 +284,8 @@ docker/docker-compose-3pd-3store-3server.yml PD exposes metrics via REST API at: - Health check: `http://:8620/actuator/health` +- Liveness: `http://:8620/v1/health` (REST listener is up) +- Readiness: `http://:8620/v1/ready` (`200` only while the PD sees a raft leader) - Metrics: `http://:8620/actuator/metrics` ## Community diff --git a/hugegraph-pd/docs/api-reference.md b/hugegraph-pd/docs/api-reference.md index aa8cce8473..33750c7ad8 100644 --- a/hugegraph-pd/docs/api-reference.md +++ b/hugegraph-pd/docs/api-reference.md @@ -774,6 +774,48 @@ curl http://localhost:8620/actuator/health } ``` +### Liveness and Readiness + +Two unauthenticated endpoints are meant for probes and startup gates: + +| Endpoint | Meaning | Status | +|----------|---------|--------| +| `GET /v1/health` | Liveness: the REST listener is up. Does not consult raft. | always `200` | +| `GET /v1/ready` | Readiness: the raft node is active and sees a leader, so this PD is inside a quorum. | `200` when ready, `503` otherwise | + +```bash +curl -i http://localhost:8620/v1/ready +``` + +**Response** (leader of a healthy cluster): +```json +{ + "ready": true, + "state": "STATE_LEADER", + "isLeader": true +} +``` + +A follower reports `"state": "STATE_FOLLOWER"` with `"isLeader": false`. When +the quorum is lost the PD keeps answering `/v1/health` with `200` but +`/v1/ready` turns into `503` with `"ready": false`. Being unauthenticated, the +body carries no cluster addresses; the leader's address stays on `/v1/members`. + +Point Kubernetes readiness probes, `depends_on` healthchecks and any +"wait for PD" script at `/v1/ready`; keep liveness probes on `/v1/health` +so a PD that merely lost its leader is not restarted. + +Match on the body rather than on the status code alone. A PD that predates this +endpoint does not reliably answer `404` for it: `RestAuthentication` refuses a +request it does not exclude by writing an error envelope, and as of 1.7.0 it +does so without setting a status, so an unknown path answers `200` with +`{"status":-1,"error":"Unauthorized!"}`. A status-only probe therefore reads +such a PD as ready. The body match holds whichever status a refusal carries: a +shell gate should use +`curl -fsS http://:8620/v1/ready | grep -q '"ready":true'`, and a +Kubernetes `httpGet` probe should be paired with a PD image that carries the +endpoint. + ### Metrics ```bash @@ -796,6 +838,21 @@ pd_store_count{state="Offline"} 0.0 pd_partition_count 36.0 ``` +#### Raft membership gauges + +Exported on `/actuator/prometheus` for alerting on quorum loss: + +| Gauge | Value | +|-------|-------| +| `hg_raft_leader` | `1` on the raft leader, `0` elsewhere | +| `hg_raft_has_leader` | `1` while this PD sees a leader (is inside a quorum), `0` otherwise | +| `hg_raft_alive_peers` | On the leader, the number of peers (itself included) heard from within the leader lease timeout (90% of the election timeout by default); `NaN` on other nodes | + +A cluster has lost its quorum when `sum(hg_raft_leader) == 0` or when +`hg_raft_has_leader == 0` on every member. Both are briefly true during a +normal election, so alert on them with a `for:` clause longer than the +election timeout rather than on the instantaneous value. + ### Partition API #### List Partitions diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftEngine.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftEngine.java index 314c9e57ef..6b744a7003 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftEngine.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftEngine.java @@ -46,6 +46,7 @@ import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.conf.Configuration; import com.alipay.sofa.jraft.core.Replicator; +import com.alipay.sofa.jraft.core.State; import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.entity.Task; import com.alipay.sofa.jraft.error.RaftError; @@ -69,7 +70,7 @@ public class RaftEngine { private PDConfig.Raft config; private RaftGroupService raftGroupService; private RpcServer rpcServer; - private Node raftNode; + private volatile Node raftNode; private RaftRpcClient raftRpcClient; public RaftEngine() { @@ -203,7 +204,101 @@ public void shutDown() { } public boolean isLeader() { - return this.raftNode.isLeader(true); + Node node = this.raftNode; + return node != null && node.isLeader(true); + } + + /** + * Whether this node currently knows a raft leader. + *

+ * A follower only keeps its leader id while heartbeats keep arriving inside the election + * timeout, and a leader only keeps its role while it can reach a quorum. A non-null leader + * therefore means this node is part of a quorum from its own point of view, which is the + * signal a readiness probe needs. + */ + public boolean hasLeader() { + return hasLeader(this.raftNode); + } + + private static boolean hasLeader(Node node) { + if (node == null) { + return false; + } + PeerId leader = node.getLeaderId(); + return leader != null && !leader.isEmpty(); + } + + /** + * Take a consistent view of the local raft state. Every field is derived from one + * {@link Node} reference and a single {@code getLeaderId()} read, so a step-down while + * the view is being built cannot report a ready node that knows no leader. + *

+ * A node is ready when it has been started, is in an active state, which jraft's + * {@code State.isActive()} takes to mean leader, transferring, candidate or follower, + * and sees a leader. Unlike a plain liveness check this turns false as soon as the + * quorum is lost. + */ + public RaftStatus getRaftStatus() { + Node node = this.raftNode; + if (node == null) { + return new RaftStatus(false, State.STATE_UNINITIALIZED.name(), false); + } + State state = node.getNodeState(); + boolean active = state != null && state.isActive(); + return new RaftStatus(active && hasLeader(node), + state == null ? State.STATE_UNINITIALIZED.name() : state.name(), + node.isLeader(true)); + } + + /** + * Immutable view of the raft state behind {@code GET /v1/ready}. It carries no cluster + * addresses: the endpoint is unauthenticated, and the leader's address stays on the + * authenticated {@code /v1/members}. + */ + public static final class RaftStatus { + + private final boolean ready; + private final String state; + private final boolean localLeader; + + RaftStatus(boolean ready, String state, boolean localLeader) { + this.ready = ready; + this.state = state; + this.localLeader = localLeader; + } + + public boolean isReady() { + return this.ready; + } + + /** + * @return the jraft node state name, never null + */ + public String getState() { + return this.state; + } + + public boolean isLocalLeader() { + return this.localLeader; + } + } + + /** + * Number of raft peers, this node included, that the leader has heard from within the + * leader lease timeout, which jraft derives as 90% of the election timeout by default. + * Only the leader tracks replication state, so any other node returns -1. + */ + public int getAlivePeerCount() { + Node node = this.raftNode; + if (node == null || !node.isLeader(true)) { + return -1; + } + try { + return node.listAlivePeers().size(); + } catch (IllegalStateException e) { + // Lost leadership between the check and the call + return -1; + } } /** @@ -232,7 +327,8 @@ public PDConfig.Raft getConfig() { } public PeerId getLeader() { - return raftNode.getLeaderId(); + Node node = this.raftNode; + return node == null ? null : node.getLeaderId(); } /** diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/metrics/PDMetrics.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/metrics/PDMetrics.java index 483974a016..54ad2a81db 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/metrics/PDMetrics.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/metrics/PDMetrics.java @@ -33,6 +33,7 @@ import org.apache.hugegraph.pd.grpc.Metapb; import org.apache.hugegraph.pd.grpc.Metapb.ShardGroup; import org.apache.hugegraph.pd.model.GraphStatistics; +import org.apache.hugegraph.pd.raft.RaftEngine; import org.apache.hugegraph.pd.service.PDRestService; import org.apache.hugegraph.pd.service.PDService; @@ -76,7 +77,28 @@ private void registerMeters() { Gauge.builder(PREFIX + ".terms", () -> setTerms()) .description("term of partitions in PD") .register(registry); + registerRaftMeters(); + } + /** + * Raft membership gauges so operators can alert on quorum loss. They mirror what + * {@code GET /v1/ready} answers: a PD that sees no leader is outside a quorum. + */ + private void registerRaftMeters() { + RaftEngine raft = RaftEngine.getInstance(); + Gauge.builder(PREFIX + ".raft.leader", () -> raft.isLeader() ? 1 : 0) + .description("1 if this PD is the raft leader, 0 otherwise") + .register(registry); + Gauge.builder(PREFIX + ".raft.has_leader", () -> raft.hasLeader() ? 1 : 0) + .description("1 if this PD sees a raft leader, i.e. is part of a quorum, 0 otherwise") + .register(registry); + Gauge.builder(PREFIX + ".raft.alive_peers", () -> { + int alive = raft.getAlivePeerCount(); + return alive < 0 ? Double.NaN : alive; + }) + .description("Number of raft peers, itself included, the leader has heard from " + + "within the leader lease timeout; NaN on non-leader nodes") + .register(registry); } private long updateGraphs() { diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/StoreAPI.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/StoreAPI.java index 2cddb29feb..b3c30ca9e1 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/StoreAPI.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/StoreAPI.java @@ -22,6 +22,7 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -31,11 +32,14 @@ import org.apache.hugegraph.pd.model.RestApiResponse; import org.apache.hugegraph.pd.model.StoreRestRequest; import org.apache.hugegraph.pd.model.TimeRangeRequest; +import org.apache.hugegraph.pd.raft.RaftEngine; import org.apache.hugegraph.pd.service.PDRestService; import org.apache.hugegraph.pd.util.DateUtil; import org.apache.hugegraph.pd.util.StoreRestAddressUtil; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -378,6 +382,10 @@ class StoreStatistics { * Check Service Health Status * This interface is used to check the health status of the service by accessing the /health * path via a GET request. + *

+ * This is a liveness signal only: it answers 200 as soon as the REST listener is up and + * does not consult the raft state. Use {@link #checkReady()} to find out whether this PD + * can actually serve. * * @return Returns a string indicating the service's health status. Typically, an empty * string indicates the service is healthy. @@ -386,4 +394,27 @@ class StoreStatistics { public Serializable checkHealthy() { return ""; } + + /** + * Check Service Readiness + * Answers 200 only when this PD is part of a raft quorum, that is, the raft node is active + * and knows the current leader. Otherwise answers 503 so that anything gating on PD + * (the compose healthcheck in front of Stores, a Kubernetes readiness probe) + * is held back until the PD can serve. Like /health this endpoint needs no authentication. + * + * @return JSON with the readiness flag, the local raft state and whether this node is the + * leader. It carries no cluster addresses, since the endpoint is unauthenticated. + */ + @GetMapping(value = "/ready", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity> checkReady() { + RaftEngine.RaftStatus status = RaftEngine.getInstance().getRaftStatus(); + + Map body = new LinkedHashMap<>(); + body.put("ready", status.isReady()); + body.put("state", status.getState()); + body.put("isLeader", status.isLocalLeader()); + HttpStatus httpStatus = status.isReady() ? HttpStatus.OK + : HttpStatus.SERVICE_UNAVAILABLE; + return ResponseEntity.status(httpStatus).body(body); + } } diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/interceptor/AuthenticationConfigurer.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/interceptor/AuthenticationConfigurer.java index 7d10416967..d4b1e026d3 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/interceptor/AuthenticationConfigurer.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/interceptor/AuthenticationConfigurer.java @@ -32,6 +32,7 @@ public class AuthenticationConfigurer implements WebMvcConfigurer { public void addInterceptors(InterceptorRegistry registry) { registry.addInterceptor(restAuthentication) .addPathPatterns("/**") - .excludePathPatterns("/actuator/*", "/v1/health", "/v1/prom/targets/*"); + .excludePathPatterns("/actuator/*", "/v1/health", "/v1/ready", + "/v1/prom/targets/*"); } } diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreSuiteTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreSuiteTest.java index 95b044c76b..4f6d3af386 100644 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreSuiteTest.java +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreSuiteTest.java @@ -22,6 +22,8 @@ import org.apache.hugegraph.pd.raft.IpAuthHandlerTest; import org.apache.hugegraph.pd.raft.RaftEngineIpAuthIntegrationTest; import org.apache.hugegraph.pd.raft.RaftEngineLeaderAddressTest; +import org.apache.hugegraph.pd.raft.RaftEngineReadinessTest; +import org.apache.hugegraph.pd.rest.interceptor.AuthenticationConfigurerTest; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -43,6 +45,8 @@ IpAuthHandlerTest.class, RaftEngineIpAuthIntegrationTest.class, RaftEngineLeaderAddressTest.class, + RaftEngineReadinessTest.class, + AuthenticationConfigurerTest.class, // StoreNodeServiceTest.class, }) @Slf4j diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/raft/RaftEngineReadinessTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/raft/RaftEngineReadinessTest.java new file mode 100644 index 0000000000..565d017f45 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/raft/RaftEngineReadinessTest.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.raft; + +import java.util.Arrays; + +import org.apache.hugegraph.testutil.Whitebox; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alipay.sofa.jraft.Node; +import com.alipay.sofa.jraft.core.State; +import com.alipay.sofa.jraft.entity.PeerId; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Covers the raft-aware readiness signal behind {@code GET /v1/ready} and the + * {@code hg.raft.*} gauges: a PD is ready only while it sees a raft leader. + */ +public class RaftEngineReadinessTest { + + private static final PeerId LEADER = new PeerId("10.0.0.1", 8610); + private static final PeerId SELF = new PeerId("10.0.0.2", 8610); + private static final PeerId OTHER = new PeerId("10.0.0.3", 8610); + + private Node originalRaftNode; + private Node mockNode; + + @Before + public void setUp() { + RaftEngine engine = RaftEngine.getInstance(); + originalRaftNode = engine.getRaftNode(); + mockNode = mock(Node.class); + Whitebox.setInternalState(engine, "raftNode", mockNode); + } + + @After + public void tearDown() { + Whitebox.setInternalState(RaftEngine.getInstance(), "raftNode", originalRaftNode); + } + + private void stub(State state, PeerId leader, boolean isLeader) { + when(mockNode.getNodeState()).thenReturn(state); + when(mockNode.getLeaderId()).thenReturn(leader); + when(mockNode.isLeader(true)).thenReturn(isLeader); + } + + @Test + public void testNotReadyBeforeRaftNodeStarts() { + Whitebox.setInternalState(RaftEngine.getInstance(), "raftNode", null); + RaftEngine engine = RaftEngine.getInstance(); + + Assert.assertFalse(engine.hasLeader()); + Assert.assertFalse(engine.isLeader()); + Assert.assertNull(engine.getLeader()); + Assert.assertEquals(-1, engine.getAlivePeerCount()); + + RaftEngine.RaftStatus status = engine.getRaftStatus(); + Assert.assertFalse(status.isReady()); + Assert.assertFalse(status.isLocalLeader()); + Assert.assertEquals(State.STATE_UNINITIALIZED.name(), status.getState()); + } + + @Test + public void testLeaderIsReady() { + stub(State.STATE_LEADER, SELF, true); + when(mockNode.listAlivePeers()).thenReturn(Arrays.asList(SELF, LEADER, OTHER)); + RaftEngine engine = RaftEngine.getInstance(); + + Assert.assertTrue(engine.hasLeader()); + Assert.assertTrue(engine.isLeader()); + Assert.assertEquals(3, engine.getAlivePeerCount()); + + RaftEngine.RaftStatus status = engine.getRaftStatus(); + Assert.assertTrue(status.isReady()); + Assert.assertTrue(status.isLocalLeader()); + Assert.assertEquals(State.STATE_LEADER.name(), status.getState()); + } + + @Test + public void testFollowerWithLeaderIsReady() { + stub(State.STATE_FOLLOWER, LEADER, false); + RaftEngine engine = RaftEngine.getInstance(); + + Assert.assertTrue(engine.getRaftStatus().isReady()); + Assert.assertTrue(engine.hasLeader()); + Assert.assertFalse(engine.isLeader()); + Assert.assertEquals(LEADER, engine.getLeader()); + // Only the leader tracks replication, followers cannot count alive peers + Assert.assertEquals(-1, engine.getAlivePeerCount()); + } + + @Test + public void testFollowerWithoutLeaderIsNotReady() { + // jraft resets the leader id once heartbeats stop arriving inside the election timeout + stub(State.STATE_FOLLOWER, null, false); + RaftEngine engine = RaftEngine.getInstance(); + + Assert.assertFalse(engine.getRaftStatus().isReady()); + Assert.assertFalse(engine.hasLeader()); + } + + @Test + public void testEmptyLeaderIdIsNotReady() { + // Defensive: NodeImpl.getLeaderId() maps an empty peer to null, so this guards the + // Node contract rather than a value the real implementation returns + stub(State.STATE_FOLLOWER, PeerId.emptyPeer(), false); + RaftEngine engine = RaftEngine.getInstance(); + + Assert.assertFalse(engine.getRaftStatus().isReady()); + Assert.assertFalse(engine.hasLeader()); + } + + @Test + public void testCandidateWithoutLeaderIsNotReady() { + // The only candidate shape jraft reaches: NodeImpl clears the leader id before it + // starts an election, so it is the missing leader, not the state, that holds it back + stub(State.STATE_CANDIDATE, null, false); + Assert.assertFalse(RaftEngine.getInstance().getRaftStatus().isReady()); + } + + @Test + public void testCandidateCountsAsActive() { + // Records the scope of the state check: State.isActive() is ordinal() < STATE_ERROR, + // so a candidate is active and would read as ready if it still knew a leader + stub(State.STATE_CANDIDATE, LEADER, false); + Assert.assertTrue(RaftEngine.getInstance().getRaftStatus().isReady()); + } + + @Test + public void testTransferringLeaderIsReady() { + stub(State.STATE_TRANSFERRING, SELF, true); + Assert.assertTrue(RaftEngine.getInstance().getRaftStatus().isReady()); + } + + @Test + public void testInactiveStatesAreNotReadyEvenWithLeaderId() { + for (State state : new State[]{State.STATE_ERROR, State.STATE_UNINITIALIZED, + State.STATE_SHUTTING, State.STATE_SHUTDOWN}) { + stub(state, LEADER, false); + Assert.assertFalse("state " + state + " must not be ready", + RaftEngine.getInstance().getRaftStatus().isReady()); + } + } + + @Test + public void testAlivePeerCountSurvivesLeadershipLossRace() { + stub(State.STATE_LEADER, SELF, true); + when(mockNode.listAlivePeers()).thenThrow(new IllegalStateException("Not leader")); + + Assert.assertEquals(-1, RaftEngine.getInstance().getAlivePeerCount()); + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/rest/RestApiTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/rest/RestApiTest.java index fb2b71d480..a145b788a2 100644 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/rest/RestApiTest.java +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/rest/RestApiTest.java @@ -62,6 +62,56 @@ public void testQueryClusterInfo() throws URISyntaxException, IOException, Inter assert obj.getInt("status") == 0; } + @Test + public void testHealthNeedsNoAuth() throws URISyntaxException, IOException, + InterruptedException { + String url = pdRestAddr + "/v1/health"; + HttpRequest request = HttpRequest.newBuilder().uri(new URI(url)).GET().build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + assert response.statusCode() == 200; + // A 200 alone does not prove the path is anonymous: as of 1.7.0 the auth interceptor + // refuses with 200 and an error envelope. checkHealthy() returns an empty body, which + // separates the two whichever status a refusal carries. + assert response.body().isEmpty() : "expected an empty body, got " + response.body(); + } + + @Test + public void testReadyNeedsNoAuthAndReflectsRaft() throws URISyntaxException, IOException, + InterruptedException, JSONException { + // The CI PD is a single-node raft group, so it is its own leader and must be ready + String url = pdRestAddr + "/v1/ready"; + HttpRequest request = HttpRequest.newBuilder().uri(new URI(url)).GET().build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + assert response.statusCode() == 200 : "expected 200, got " + response.statusCode() + + " body=" + response.body(); + JSONObject obj = new JSONObject(response.body()); + assert obj.getBoolean("ready"); + assert obj.getBoolean("isLeader"); + assert "STATE_LEADER".equals(obj.getString("state")); + // Unauthenticated, so it must not disclose cluster addresses + assert !obj.has("leader") : "the anonymous body must not carry the leader address"; + } + + @Test + public void testRaftGaugesExported() throws URISyntaxException, IOException, + InterruptedException { + String url = pdRestAddr + "/actuator/prometheus"; + HttpRequest request = HttpRequest.newBuilder().uri(new URI(url)).GET().build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + assert response.statusCode() == 200; + String body = response.body(); + assert body.contains("hg_raft_leader{") : "missing hg_raft_leader gauge"; + assert body.contains("hg_raft_has_leader{") : "missing hg_raft_has_leader gauge"; + assert body.contains("hg_raft_alive_peers{") : "missing hg_raft_alive_peers gauge"; + // Single-node CI cluster: this PD is the leader and hears from itself + assert body.matches("(?s).*hg_raft_leader\\{[^}]*\\} 1\\.0.*") : + "hg_raft_leader should be 1 on a single-node leader"; + assert body.matches("(?s).*hg_raft_has_leader\\{[^}]*\\} 1\\.0.*") : + "hg_raft_has_leader should be 1 on a single-node leader"; + assert body.matches("(?s).*hg_raft_alive_peers\\{[^}]*\\} 1\\.0.*") : + "hg_raft_alive_peers should be 1 on a single-node leader"; + } + @Test public void testQueryClusterMembers() throws URISyntaxException, IOException, InterruptedException, JSONException { diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/rest/interceptor/AuthenticationConfigurerTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/rest/interceptor/AuthenticationConfigurerTest.java new file mode 100644 index 0000000000..726b7df816 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/rest/interceptor/AuthenticationConfigurerTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.rest.interceptor; + +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.springframework.util.AntPathMatcher; +import org.springframework.util.PathMatcher; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; +import org.springframework.web.servlet.handler.MappedInterceptor; + +import static org.mockito.Mockito.mock; + +/** + * The probe endpoints have to stay outside the auth interceptor. If one of them slips back + * behind it, PD answers a probe with 200 and an error envelope instead of a readiness + * answer, so every healthcheck gating on the body holds forever while the status still + * looks fine. That failure is silent, hence a check here rather than only in the live + * REST suite. + */ +public class AuthenticationConfigurerTest { + + private static final PathMatcher MATCHER = new AntPathMatcher(); + + /** + * {@code InterceptorRegistry.getInterceptors()} is protected, so read it from a subclass. + */ + private static final class TestRegistry extends InterceptorRegistry { + + List registered() { + return getInterceptors(); + } + } + + private static MappedInterceptor authInterceptor() { + AuthenticationConfigurer configurer = new AuthenticationConfigurer(); + configurer.restAuthentication = mock(RestAuthentication.class); + + TestRegistry registry = new TestRegistry(); + configurer.addInterceptors(registry); + + List registered = registry.registered(); + Assert.assertEquals(1, registered.size()); + return (MappedInterceptor) registered.get(0); + } + + @Test + public void testProbeEndpointsAreAnonymous() { + // A kubelet probe and a compose healthcheck cannot present credentials + MappedInterceptor auth = authInterceptor(); + Assert.assertFalse("/v1/ready must not be intercepted", + auth.matches("/v1/ready", MATCHER)); + Assert.assertFalse("/v1/health must not be intercepted", + auth.matches("/v1/health", MATCHER)); + Assert.assertFalse("/actuator/prometheus must not be intercepted", + auth.matches("/actuator/prometheus", MATCHER)); + } + + @Test + public void testEverythingElseStaysAuthenticated() { + MappedInterceptor auth = authInterceptor(); + Assert.assertTrue("/v1/members carries cluster addresses and must stay authenticated", + auth.matches("/v1/members", MATCHER)); + Assert.assertTrue(auth.matches("/v1/stores", MATCHER)); + Assert.assertTrue(auth.matches("/v1/members/change", MATCHER)); + } +} diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh index ab73255b8c..0a78c30da2 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh @@ -111,6 +111,19 @@ wait_for_pd() { return 1 } +# Wait until PD readiness answers, or timeout. This is the gate the docs recommend: +# -f rejects the 503, the body match rejects a 200 that is an auth envelope rather +# than a readiness answer. No credentials, and a single-node group elects itself. +wait_for_pd_ready() { + local elapsed=0 + while (( elapsed < STARTUP_WAIT )); do + curl -fsS "$PD_URL/v1/ready" 2>/dev/null | grep -q '"ready":true' && return 0 + sleep 2 + elapsed=$((elapsed + 2)) + done + return 1 +} + # Wait until bin/pid is non-empty or timeout wait_for_pid_file() { local elapsed=0 @@ -199,6 +212,13 @@ else fail "PD health endpoint not responding after ${STARTUP_WAIT}s" fi +info "Waiting up to ${STARTUP_WAIT}s for PD readiness endpoint..." +if wait_for_pd_ready; then + pass "PD readiness endpoint reports a raft leader at $PD_URL/v1/ready" +else + fail "PD readiness endpoint did not report ready=true after ${STARTUP_WAIT}s" +fi + cleanup # ── test 2: foreground mode blocks ─────────────────────────────────────────── diff --git a/hugegraph-store/docs/deployment-guide.md b/hugegraph-store/docs/deployment-guide.md index de07904d64..f002046927 100644 --- a/hugegraph-store/docs/deployment-guide.md +++ b/hugegraph-store/docs/deployment-guide.md @@ -719,10 +719,15 @@ environment: ``` **Startup ordering** is enforced via `depends_on` with `condition: service_healthy`: -1. PD nodes start first and must pass healthchecks (`/v1/health`) +1. PD nodes start first and must pass healthchecks (`/v1/health`, liveness only) 2. Store nodes start after all PD nodes are healthy 3. Server nodes start after all Store nodes are healthy +`/v1/health` answers `200` as soon as the PD REST listener is up, so step 1 does +not wait for a raft quorum to form. PD also serves `/v1/ready`, which answers +`200` only while the PD sees a raft leader; `docker/README.md` covers what +pointing the healthchecks at it requires. + > **Note**: The deprecated env var names (`GRPC_HOST`, `RAFT_ADDRESS`, `RAFT_PEERS`, `PD_ADDRESS`, `BACKEND`, `PD_PEERS`) still work but log a warning. Use the `HG_*` prefixed names for new deployments. **Deploy**: @@ -855,9 +860,12 @@ kubectl port-forward svc/hugegraph-store 8500:8500 -n hugegraph ### Health Check ```bash -# PD health +# PD liveness (REST listener up) curl http://192.168.1.10:8620/v1/health +# PD readiness (200 only while the PD sees a raft leader, 503 otherwise) +curl -i http://192.168.1.10:8620/v1/ready + # Store health curl http://192.168.1.20:8520/v1/health ```