From 371b9f0083ee541039be71e6fdb6b672ece2ea72 Mon Sep 17 00:00:00 2001 From: asdf2014 Date: Tue, 7 Jul 2026 13:06:17 +0800 Subject: [PATCH] ZOOKEEPER-2789: Reassign ZXID to a 24-bit epoch / 40-bit counter with rolling upgrade Merge current master into the reassign_zxid branch and rework the change into an opt-in, rolling-upgradeable wide-counter zxid layout. The 32-bit counter overflowing forces a leader re-election roughly every 49.7 days at 1k writes/s (ZOOKEEPER-1277). This adds a 24-bit epoch / 40-bit counter layout that pushes that to ~34.9 years while still allowing ~1915 years of hourly elections before the epoch overflows. - ZxidLayout / ZxidLayoutState: the legacy 32/32 and wide 24/40 layouts plus a per-QuorumPeer record of the epoch from which the ensemble is wide. The static ZxidUtils helpers remain the legacy-fixed layout for the on-the-wire epoch carrier zxids of the handshake. - Opt-in via zookeeper.wideCounterZxidEnabled (default off). A newly elected leader switches at its new epoch, persists it to a new zxidLayoutSwitchEpoch file, and announces it in the LEADERINFO handshake (learner protocol version bumped to 0x11000). Learners below 0x11000 are refused once switched. - The switch coincides with an epoch bump, so the numeric order of zxids is preserved; layoutFor(zxid) selects the decode layout per zxid so existing legacy snapshots/txnlogs are read correctly across the boundary without being rewritten. One-way switch with an anti-downgrade guard. - Replace hardcoded zxid bit operations with layout-aware calls across Leader, LearnerHandler, Learner, Follower, ObserverMaster, ZKDatabase, FastLeaderElection, Commands and LogChopper. --- .../apache/zookeeper/server/ZKDatabase.java | 11 +- .../zookeeper/server/admin/Commands.java | 6 +- .../server/persistence/TxnLogToolkit.java | 30 ++- .../server/quorum/FastLeaderElection.java | 3 +- .../zookeeper/server/quorum/Follower.java | 2 +- .../quorum/FollowerZooKeeperServer.java | 2 +- .../zookeeper/server/quorum/Leader.java | 53 ++++- .../zookeeper/server/quorum/Learner.java | 33 +++- .../server/quorum/LearnerHandler.java | 69 ++++++- .../server/quorum/LearnerMaster.java | 7 + .../server/quorum/ObserverMaster.java | 8 +- .../zookeeper/server/quorum/QuorumPeer.java | 113 ++++++++++- .../zookeeper/server/util/LogChopper.java | 17 +- .../zookeeper/server/util/ZxidLayout.java | 102 ++++++++++ .../server/util/ZxidLayoutState.java | 162 +++++++++++++++ .../zookeeper/server/ZxidRolloverTest.java | 5 +- .../server/persistence/TxnLogToolkitTest.java | 95 +++++++-- .../FLEMalformedNotificationMessageTest.java | 41 ++++ .../server/quorum/LearnerHandlerTest.java | 63 ++++++ .../quorum/QuorumPeerZxidLayoutTest.java | 173 ++++++++++++++++ .../quorum/WideCounterZxidUpgradeTest.java | 153 ++++++++++++++ .../zookeeper/server/quorum/Zab1_0Test.java | 187 ++++++++++++++++-- .../zookeeper/server/util/LogChopperTest.java | 105 ++++++++++ .../server/util/ZxidLayoutStateTest.java | 137 +++++++++++++ .../zookeeper/server/util/ZxidLayoutTest.java | 128 ++++++++++++ .../zookeeper/server/util/ZxidUtilsTest.java | 80 ++++++++ .../test/FollowerResyncConcurrencyTest.java | 5 +- .../apache/zookeeper/test/ReconfigTest.java | 3 +- 28 files changed, 1719 insertions(+), 74 deletions(-) create mode 100644 zookeeper-server/src/main/java/org/apache/zookeeper/server/util/ZxidLayout.java create mode 100644 zookeeper-server/src/main/java/org/apache/zookeeper/server/util/ZxidLayoutState.java create mode 100644 zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerZxidLayoutTest.java create mode 100644 zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/WideCounterZxidUpgradeTest.java create mode 100644 zookeeper-server/src/test/java/org/apache/zookeeper/server/util/LogChopperTest.java create mode 100644 zookeeper-server/src/test/java/org/apache/zookeeper/server/util/ZxidLayoutStateTest.java create mode 100644 zookeeper-server/src/test/java/org/apache/zookeeper/server/util/ZxidLayoutTest.java create mode 100644 zookeeper-server/src/test/java/org/apache/zookeeper/server/util/ZxidUtilsTest.java diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZKDatabase.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZKDatabase.java index 2fd67da5063..89df90833fd 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZKDatabase.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZKDatabase.java @@ -58,6 +58,7 @@ import org.apache.zookeeper.server.quorum.Leader.PureRequestProposal; import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; import org.apache.zookeeper.server.util.SerializeUtils; +import org.apache.zookeeper.server.util.ZxidLayoutState; import org.apache.zookeeper.server.util.ZxidUtils; import org.apache.zookeeper.txn.TxnDigest; import org.apache.zookeeper.txn.TxnHeader; @@ -85,6 +86,13 @@ public class ZKDatabase { private final boolean allowDiscontinuousProposals = Boolean.getBoolean("zookeeper.test.allowDiscontinuousProposals"); + /** How the zxids in this database are split into epoch and counter; set by the owning QuorumPeer. */ + private volatile ZxidLayoutState zxidLayoutState = ZxidLayoutState.legacyOnly(); + + public void setZxidLayoutState(ZxidLayoutState zxidLayoutState) { + this.zxidLayoutState = zxidLayoutState; + } + /** * Default value is to use snapshot if txnlog size exceeds 1/3 the size of snapshot */ @@ -332,7 +340,8 @@ public void addCommittedProposal(Request request) { return; } else if (!allowDiscontinuousProposals && request.zxid != maxCommittedLog + 1 - && ZxidUtils.getEpochFromZxid(request.zxid) <= ZxidUtils.getEpochFromZxid(maxCommittedLog)) { + && zxidLayoutState.layoutFor(request.zxid).getEpochFromZxid(request.zxid) + <= zxidLayoutState.layoutFor(maxCommittedLog).getEpochFromZxid(maxCommittedLog)) { String msg = String.format( "Committed proposal cached out of order: 0x%s is not the next proposal of 0x%s", ZxidUtils.zxidToString(request.zxid), diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java index af69008986c..03dc7d2a5d2 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java @@ -71,6 +71,7 @@ import org.apache.zookeeper.server.quorum.ReadOnlyZooKeeperServer; import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; import org.apache.zookeeper.server.util.RateLimiter; +import org.apache.zookeeper.server.util.ZxidLayout; import org.apache.zookeeper.server.util.ZxidUtils; import org.eclipse.jetty.http.HttpStatus; import org.slf4j.Logger; @@ -1211,8 +1212,9 @@ public CommandResponse runGet(ZooKeeperServer zkServer, Map kwar response.put("voting", voting); long lastProcessedZxid = zkServer.getZKDatabase().getDataTreeLastProcessedZxid(); response.put("last_zxid", "0x" + ZxidUtils.zxidToString(lastProcessedZxid)); - response.put("zab_epoch", ZxidUtils.getEpochFromZxid(lastProcessedZxid)); - response.put("zab_counter", ZxidUtils.getCounterFromZxid(lastProcessedZxid)); + ZxidLayout layout = peer.getZxidLayoutState().layoutFor(lastProcessedZxid); + response.put("zab_epoch", layout.getEpochFromZxid(lastProcessedZxid)); + response.put("zab_counter", layout.getCounterFromZxid(lastProcessedZxid)); response.put("zabstate", zabState.name().toLowerCase()); } else { response.put("voting", false); diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/TxnLogToolkit.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/TxnLogToolkit.java index 8cd08212254..30c93488436 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/TxnLogToolkit.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/TxnLogToolkit.java @@ -55,6 +55,7 @@ import org.apache.zookeeper.server.TxnLogEntry; import org.apache.zookeeper.server.util.LogChopper; import org.apache.zookeeper.server.util.SerializeUtils; +import org.apache.zookeeper.server.util.ZxidLayoutState; import org.apache.zookeeper.txn.CheckVersionTxn; import org.apache.zookeeper.txn.CreateContainerTxn; import org.apache.zookeeper.txn.CreateTTLTxn; @@ -117,6 +118,12 @@ Options getOptions() { // chop mode private long zxid = -1L; + // chop mode: the epoch from which the ensemble uses the wide-counter zxid + // layout (the content of the zxidLayoutSwitchEpoch file), or -1 when the + // log is entirely in the legacy layout. Only affects the gap diagnostics, + // which decompose zxids into epoch/counter; the chop boundary itself is a + // purely numeric comparison and is layout-independent. + private long zxidLayoutSwitchEpoch = -1L; /** * @param args Command line arguments @@ -164,8 +171,15 @@ public TxnLogToolkit( } public TxnLogToolkit(String txnLogFileName, String zxidName) throws TxnLogToolkitException { + this(txnLogFileName, zxidName, null); + } + + public TxnLogToolkit(String txnLogFileName, String zxidName, String switchEpochName) throws TxnLogToolkitException { txnLogFile = loadTxnFile(txnLogFileName); zxid = Long.decode(zxidName); + if (switchEpochName != null) { + zxidLayoutSwitchEpoch = Long.decode(switchEpochName); + } } private File loadTxnFile(String txnLogFileName) throws TxnLogToolkitException { @@ -253,9 +267,14 @@ public void dump(Scanner scanner) throws Exception { public void chop() { File targetFile = new File(txnLogFile.getParentFile(), txnLogFile.getName() + ".chopped" + zxid); + ZxidLayoutState layoutState = ZxidLayoutState.legacyOnly(); + if (zxidLayoutSwitchEpoch >= 0) { + layoutState = new ZxidLayoutState(); + layoutState.switchAt(zxidLayoutSwitchEpoch); + } try (InputStream is = new BufferedInputStream(new FileInputStream(txnLogFile)); OutputStream os = new BufferedOutputStream(new FileOutputStream(targetFile))) { - if (!LogChopper.chop(is, os, zxid)) { + if (!LogChopper.chop(is, os, zxid, layoutState)) { throw new TxnLogToolkitException( ExitCode.INVALID_INVOCATION.getValue(), "Failed to chop %s", @@ -457,8 +476,13 @@ private static TxnLogToolkit parseCommandLine(String[] args) throws TxnLogToolki // Chop mode options Option chopOpt = new Option("c", "chop", false, "Chop mode. Chop txn file to a zxid."); Option zxidOpt = new Option("z", "zxid", true, "Used with chop. Zxid to which to chop."); + Option switchEpochOpt = new Option("s", "switch-epoch", true, + "Used with chop. The wide-counter zxid layout switch epoch (content of the zxidLayoutSwitchEpoch " + + "file) when the log spans the layout switch, so gap diagnostics decode zxids correctly. " + + "Optional; the chop result is the same with or without it."); options.addOption(chopOpt); options.addOption(zxidOpt); + options.addOption(switchEpochOpt); try { CommandLine cli = parser.parse(options, args); @@ -469,7 +493,7 @@ private static TxnLogToolkit parseCommandLine(String[] args) throws TxnLogToolki printHelpAndExit(1, options); } if (cli.hasOption("chop") && cli.hasOption("zxid")) { - return new TxnLogToolkit(cli.getArgs()[0], cli.getOptionValue("zxid")); + return new TxnLogToolkit(cli.getArgs()[0], cli.getOptionValue("zxid"), cli.getOptionValue("switch-epoch")); } return new TxnLogToolkit(cli.hasOption("recover"), cli.hasOption("verbose"), cli.getArgs()[0], cli.hasOption("yes")); } catch (ParseException e) { @@ -479,7 +503,7 @@ private static TxnLogToolkit parseCommandLine(String[] args) throws TxnLogToolki private static void printHelpAndExit(int exitCode, Options options) { HelpFormatter help = new HelpFormatter(); - help.printHelp(120, "TxnLogToolkit [-dhrvc] (-z )", "", options, ""); + help.printHelp(120, "TxnLogToolkit [-dhrvc] (-z ) [-s ]", "", options, ""); ServiceUtils.requestSystemExit(exitCode); } diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FastLeaderElection.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FastLeaderElection.java index 61c0eb60104..5be2271ecae 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FastLeaderElection.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FastLeaderElection.java @@ -36,7 +36,6 @@ import org.apache.zookeeper.server.quorum.QuorumPeer.ServerState; import org.apache.zookeeper.server.quorum.flexible.QuorumOracleMaj; import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; -import org.apache.zookeeper.server.util.ZxidUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -284,7 +283,7 @@ public void run() { } } else { LOG.info("Backward compatibility mode (28 bits), server id: {}", response.sid); - rpeerepoch = ZxidUtils.getEpochFromZxid(rzxid); + rpeerepoch = self.getZxidLayoutState().layoutFor(rzxid).getEpochFromZxid(rzxid); } // check if we have a version that includes config. If so extract config info from message. diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Follower.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Follower.java index ca99974cb52..07f75338c80 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Follower.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Follower.java @@ -93,7 +93,7 @@ void followLeader() throws InterruptedException { } //check to see if the leader zxid is lower than ours //this should never happen but is just a safety check - long newEpoch = ZxidUtils.getEpochFromZxid(newEpochZxid); + long newEpoch = self.getZxidLayoutState().current().getEpochFromZxid(newEpochZxid); if (newEpoch < self.getAcceptedEpoch()) { LOG.error("Proposed leader epoch " + ZxidUtils.zxidToString(newEpochZxid) diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java index 1b0b5cd9276..2c6ded3e121 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java @@ -77,7 +77,7 @@ protected void setupRequestProcessors() { LinkedBlockingQueue pendingTxns = new LinkedBlockingQueue<>(); public void logRequest(Request request) { - if ((request.zxid & 0xffffffffL) != 0) { + if (self.getZxidLayoutState().current().getCounterFromZxid(request.zxid) != 0) { pendingTxns.add(request); } syncProcessor.processRequest(request); diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Leader.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Leader.java index 1561a7dae18..f111d90b99c 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Leader.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Leader.java @@ -68,7 +68,8 @@ import org.apache.zookeeper.server.quorum.QuorumPeer.LearnerType; import org.apache.zookeeper.server.quorum.auth.QuorumAuthServer; import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; -import org.apache.zookeeper.server.util.ZxidUtils; +import org.apache.zookeeper.server.util.ZxidLayout; +import org.apache.zookeeper.server.util.ZxidLayoutState; import org.apache.zookeeper.util.ServiceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -654,7 +655,7 @@ void lead() throws IOException, InterruptedException { long epoch = getEpochToPropose(self.getMyId(), self.getAcceptedEpoch()); - zk.setZxid(ZxidUtils.makeZxid(epoch, 0)); + zk.setZxid(self.getZxidLayoutState().current().makeZxid(epoch, 0)); synchronized (this) { lastProposed = zk.getZxid(); @@ -662,7 +663,7 @@ void lead() throws IOException, InterruptedException { newLeaderProposal.packet = new QuorumPacket(NEWLEADER, zk.getZxid(), null, null); - if ((newLeaderProposal.packet.getZxid() & 0xffffffffL) != 0) { + if (self.getZxidLayoutState().current().getCounterFromZxid(newLeaderProposal.packet.getZxid()) != 0) { LOG.info("NEWLEADER proposal has Zxid of {}", Long.toHexString(newLeaderProposal.packet.getZxid())); } @@ -755,7 +756,7 @@ void lead() throws IOException, InterruptedException { String initialZxid = System.getProperty("zookeeper.testingonly.initialZxid"); if (initialZxid != null) { long zxid = Long.parseLong(initialZxid); - zk.setZxid((zk.getZxid() & 0xffffffff00000000L) | zxid); + zk.setZxid(self.getZxidLayoutState().current().clearCounter(zk.getZxid()) | zxid); } if (!System.getProperty("zookeeper.leaderServes", "yes").equals("no")) { @@ -1065,7 +1066,7 @@ public synchronized void processAck(long sid, long zxid, SocketAddress followerA LOG.trace("outstanding proposals all"); } - if ((zxid & 0xffffffffL) == 0) { + if (self.getZxidLayoutState().current().getCounterFromZxid(zxid) == 0) { /* * We no longer process NEWLEADER ack with this method. However, * the learner sends an ack back to the leader after it gets @@ -1274,7 +1275,15 @@ public synchronized long getLastProposed() { * Returns the current epoch of the leader. */ public long getEpoch() { - return ZxidUtils.getEpochFromZxid(lastProposed); + // Read lastProposed through the synchronized getter: everywhere else + // it is accessed under the Leader monitor, so reading it unsynchronized + // here would be an inconsistent-synchronization data race. + return self.getZxidLayoutState().current().getEpochFromZxid(getLastProposed()); + } + + @Override + public ZxidLayoutState getZxidLayoutState() { + return self.getZxidLayoutState(); } @SuppressWarnings("serial") @@ -1298,11 +1307,13 @@ public Proposal propose(Request request) throws XidRolloverException { ServiceUtils.requestSystemExit(ExitCode.UNEXPECTED_ERROR.getValue()); } /** - * Address the rollover issue. All lower 32bits set indicate a new leader - * election. Force a re-election instead. See ZOOKEEPER-1277 + * Address the rollover issue. An exhausted counter part indicates a + * new leader election. Force a re-election instead. See ZOOKEEPER-1277 */ - if ((request.zxid & 0xffffffffL) == 0xffffffffL) { - String msg = "zxid lower 32 bits have rolled over, forcing re-election, and therefore new epoch start"; + ZxidLayout layout = self.getZxidLayoutState().current(); + if (layout.getCounterFromZxid(request.zxid) == layout.getMaxCounter()) { + String msg = "zxid counter (lower " + layout.getCounterBits() + + " bits) has rolled over, forcing re-election, and therefore new epoch start"; shutdown(msg); throw new XidRolloverException(msg); } @@ -1480,6 +1491,7 @@ public long getEpochToPropose(long sid, long lastAcceptedEpoch) throws Interrupt QuorumVerifier verifier = self.getQuorumVerifier(); if (connectingFollowers.contains(self.getMyId()) && verifier.containsQuorum(connectingFollowers)) { waitingForNewEpoch = false; + maybeSwitchZxidLayout(epoch); self.setAcceptedEpoch(epoch); connectingFollowers.notifyAll(); } else { @@ -1501,6 +1513,27 @@ public long getEpochToPropose(long sid, long lastAcceptedEpoch) throws Interrupt } } + /** + * Switches the ensemble to the wide-counter zxid layout at the moment + * the new epoch is established, if enabled and not yet switched. Doing + * it inside the epoch decision point guarantees that every zxid of the + * new epoch — the leader's own as well as those handed to the + * LearnerHandlers — is composed with one consistent layout. + */ + private void maybeSwitchZxidLayout(long newEpoch) throws IOException { + if (!self.isWideCounterZxidEnabled() || self.getZxidLayoutState().isSwitched()) { + return; + } + if (newEpoch > ZxidLayout.WIDE_COUNTER.getMaxEpoch()) { + LOG.warn("Cannot switch to the {} zxid layout: epoch 0x{} exceeds its largest representable epoch 0x{}", + ZxidLayout.WIDE_COUNTER, + Long.toHexString(newEpoch), + Long.toHexString(ZxidLayout.WIDE_COUNTER.getMaxEpoch())); + return; + } + self.recordZxidLayoutSwitch(newEpoch); + } + @Override public ZKDatabase getZKDatabase() { return zk.getZKDatabase(); diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Learner.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Learner.java index adf0ef6e510..1396515f1e3 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Learner.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Learner.java @@ -61,6 +61,7 @@ import org.apache.zookeeper.server.util.ConfigUtils; import org.apache.zookeeper.server.util.MessageTracker; import org.apache.zookeeper.server.util.SerializeUtils; +import org.apache.zookeeper.server.util.ZxidLayoutState; import org.apache.zookeeper.server.util.ZxidUtils; import org.apache.zookeeper.txn.SetDataTxn; import org.apache.zookeeper.txn.TxnDigest; @@ -490,12 +491,15 @@ protected long registerWithLeader(int pktType) throws IOException { long lastLoggedZxid = self.getLastLoggedZxid(); QuorumPacket qp = new QuorumPacket(); qp.setType(pktType); + // The zxid of this packet is only a carrier for the accepted epoch + // and stays in the legacy layout on the wire, whatever layout the + // data uses; so does the zxid of the LEADERINFO reply. qp.setZxid(ZxidUtils.makeZxid(self.getAcceptedEpoch(), 0)); /* * Add sid to payload */ - LearnerInfo li = new LearnerInfo(self.getMyId(), 0x10000, self.getQuorumVerifier().getVersion()); + LearnerInfo li = new LearnerInfo(self.getMyId(), LearnerHandler.WIDE_COUNTER_PROTOCOL_VERSION, self.getQuorumVerifier().getVersion()); ByteArrayOutputStream bsid = new ByteArrayOutputStream(); BinaryOutputArchive boa = BinaryOutputArchive.getArchive(bsid); boa.writeRecord(li, "LearnerInfo"); @@ -506,7 +510,18 @@ protected long registerWithLeader(int pktType) throws IOException { final long newEpoch = ZxidUtils.getEpochFromZxid(qp.getZxid()); if (qp.getType() == Leader.LEADERINFO) { // we are connected to a 1.0 server so accept the new epoch and read the next packet - leaderProtocolVersion = ByteBuffer.wrap(qp.getData()).getInt(); + ByteBuffer leaderInfoData = ByteBuffer.wrap(qp.getData()); + leaderProtocolVersion = leaderInfoData.getInt(); + if (leaderInfoData.remaining() >= 8) { + // The leader runs the ensemble in the wide-counter zxid + // layout from the announced epoch on; record it before + // parsing any zxid of the new epoch. + self.adoptZxidLayoutSwitch(leaderInfoData.getLong()); + } else { + // The leader announced no layout switch; forget a recorded + // one if it never made it into any data (see there). + self.abandonUnusedZxidLayoutSwitch(); + } byte[] epochBytes = new byte[4]; final ByteBuffer wrappedEpochBytes = ByteBuffer.wrap(epochBytes); if (newEpoch > self.getAcceptedEpoch()) { @@ -526,7 +541,10 @@ protected long registerWithLeader(int pktType) throws IOException { } QuorumPacket ackNewEpoch = new QuorumPacket(Leader.ACKEPOCH, lastLoggedZxid, epochBytes, null); writePacket(ackNewEpoch, true); - return ZxidUtils.makeZxid(newEpoch, 0); + // Unlike the epoch carrier zxids above, the returned zxid is a + // real one (the first zxid of the new epoch), composed with the + // layout just adopted from the leader. + return self.getZxidLayoutState().current().makeZxid(newEpoch, 0); } else { if (newEpoch > self.getAcceptedEpoch()) { self.setAcceptedEpoch(newEpoch); @@ -543,7 +561,10 @@ long enforceContinuousProposal(long lastQueued, PacketInFlight pif) throws Excep if (lastQueued == 0) { LOG.info("DIFF sync got first proposal 0x{}", Long.toHexString(pif.hdr.getZxid())); } else if (pif.hdr.getZxid() != lastQueued + 1) { - if (ZxidUtils.getEpochFromZxid(pif.hdr.getZxid()) <= ZxidUtils.getEpochFromZxid(lastQueued)) { + // A DIFF may span the layout switch epoch, so pick the layout per zxid. + ZxidLayoutState layoutState = self.getZxidLayoutState(); + if (layoutState.layoutFor(pif.hdr.getZxid()).getEpochFromZxid(pif.hdr.getZxid()) + <= layoutState.layoutFor(lastQueued).getEpochFromZxid(lastQueued)) { String msg = String.format( "DIFF sync got proposal 0x%s, last queued 0x%s, expected 0x%s", Long.toHexString(pif.hdr.getZxid()), Long.toHexString(lastQueued), @@ -570,7 +591,7 @@ long enforceContinuousProposal(long lastQueued, PacketInFlight pif) throws Excep protected void syncWithLeader(long newLeaderZxid) throws Exception { QuorumPacket ack = new QuorumPacket(Leader.ACK, 0, null, null); QuorumPacket qp = new QuorumPacket(); - long newEpoch = ZxidUtils.getEpochFromZxid(newLeaderZxid); + long newEpoch = self.getZxidLayoutState().current().getEpochFromZxid(newLeaderZxid); QuorumVerifier newLeaderQV = null; @@ -822,7 +843,7 @@ protected void syncWithLeader(long newLeaderZxid) throws Exception { } } } - ack.setZxid(ZxidUtils.makeZxid(newEpoch, 0)); + ack.setZxid(self.getZxidLayoutState().current().makeZxid(newEpoch, 0)); writePacket(ack, true); zk.startup(); /* diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LearnerHandler.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LearnerHandler.java index 57947daa86c..f353e58a620 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LearnerHandler.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LearnerHandler.java @@ -54,6 +54,8 @@ import org.apache.zookeeper.server.quorum.auth.QuorumAuthServer; import org.apache.zookeeper.server.util.ConfigUtils; import org.apache.zookeeper.server.util.MessageTracker; +import org.apache.zookeeper.server.util.ZxidLayout; +import org.apache.zookeeper.server.util.ZxidLayoutState; import org.apache.zookeeper.server.util.ZxidUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,6 +109,15 @@ String getRemoteAddress() { protected int version = 0x1; + /** + * The learner protocol version that understands the wide-counter zxid + * layout (ZOOKEEPER-2789): the LEADERINFO payload may carry the layout + * switch epoch, and every zxid from the switch epoch on is composed of + * a 24-bit epoch and a 40-bit counter. Learners below this version are + * refused once the ensemble has switched. + */ + public static final int WIDE_COUNTER_PROTOCOL_VERSION = 0x11000; + int getVersion() { return version; } @@ -517,24 +528,63 @@ public void run() { learnerMaster.registerLearnerHandlerBean(this, sock); + // The zxid of a FOLLOWERINFO / OBSERVERINFO packet is only a + // carrier for the accepted epoch and stays in the legacy layout + // on the wire, whatever layout the data uses. long lastAcceptedEpoch = ZxidUtils.getEpochFromZxid(qp.getZxid()); long peerLastZxid; StateSummary ss = null; long zxid = qp.getZxid(); long newEpoch = learnerMaster.getEpochToPropose(this.getSid(), lastAcceptedEpoch); - long newLeaderZxid = ZxidUtils.makeZxid(newEpoch, 0); + // The layout switch, if any, happened inside getEpochToPropose(), + // so the layout is settled by now. + ZxidLayoutState layoutState = learnerMaster.getZxidLayoutState(); + long newLeaderZxid = layoutState.current().makeZxid(newEpoch, 0); + + // Once the ensemble has switched to the wide-counter zxid layout, + // every zxid from the switch epoch on is composed of a 24-bit + // epoch and a 40-bit counter. A learner announcing a protocol + // version below WIDE_COUNTER_PROTOCOL_VERSION cannot decode those + // zxids and must be refused before it is advanced through + // syncFollower / NEWLEADER — checked here, above the handshake + // branch split, so it also covers the pre-0x10000 legacy branch + // (which never receives a LEADERINFO carrying the switch epoch). + if (layoutState.isSwitched() && this.getVersion() < WIDE_COUNTER_PROTOCOL_VERSION) { + LOG.error( + "Refusing learner sid: {} with protocol version 0x{}: this ensemble switched to the {} zxid" + + " layout at epoch 0x{}, which needs protocol version 0x{} or above. Upgrade that member.", + this.sid, + Integer.toHexString(this.getVersion()), + ZxidLayout.WIDE_COUNTER, + Long.toHexString(layoutState.getSwitchEpoch()), + Integer.toHexString(WIDE_COUNTER_PROTOCOL_VERSION)); + return; + } if (this.getVersion() < 0x10000) { // we are going to have to extrapolate the epoch information - long epoch = ZxidUtils.getEpochFromZxid(zxid); + long epoch = layoutState.layoutFor(zxid).getEpochFromZxid(zxid); ss = new StateSummary(epoch, zxid); // fake the message learnerMaster.waitForEpochAck(this.getSid(), ss); } else { - byte[] ver = new byte[4]; - ByteBuffer.wrap(ver).putInt(0x10000); - QuorumPacket newEpochPacket = new QuorumPacket(Leader.LEADERINFO, newLeaderZxid, ver, null); + // Like the FOLLOWERINFO zxid, the LEADERINFO zxid is only an + // epoch carrier and stays in the legacy layout on the wire. + long epochCarrierZxid = ZxidUtils.makeZxid(newEpoch, 0); + byte[] ver; + if (layoutState.isSwitched()) { + // Announce the switch epoch so the learner parses the + // synced data and every future zxid with the right layouts. + ver = new byte[12]; + ByteBuffer verBuffer = ByteBuffer.wrap(ver); + verBuffer.putInt(WIDE_COUNTER_PROTOCOL_VERSION); + verBuffer.putLong(layoutState.getSwitchEpoch()); + } else { + ver = new byte[4]; + ByteBuffer.wrap(ver).putInt(WIDE_COUNTER_PROTOCOL_VERSION); + } + QuorumPacket newEpochPacket = new QuorumPacket(Leader.LEADERINFO, epochCarrierZxid, ver, null); oa.writeRecord(newEpochPacket, "packet"); messageTracker.trackSent(Leader.LEADERINFO); bufferedOutput.flush(); @@ -789,7 +839,8 @@ boolean syncFollower(long peerLastZxid, LearnerMaster learnerMaster) { * zxid in our history. In this case, we will ignore TRUNC logic and * always send DIFF if we have old enough history */ - boolean isPeerNewEpochZxid = (peerLastZxid & 0xffffffffL) == 0; + ZxidLayoutState layoutState = learnerMaster.getZxidLayoutState(); + boolean isPeerNewEpochZxid = layoutState.layoutFor(peerLastZxid).getCounterFromZxid(peerLastZxid) == 0; // Keep track of the latest zxid which already queued long currentZxid = peerLastZxid; boolean needSnap = true; @@ -949,7 +1000,8 @@ boolean syncFollower(long peerLastZxid, LearnerMaster learnerMaster) { * @return last zxid of the queued proposal */ protected long queueCommittedProposals(Iterator itr, long peerLastZxid, Long maxZxid, Long lastCommittedZxid) { - boolean isPeerNewEpochZxid = (peerLastZxid & 0xffffffffL) == 0; + ZxidLayoutState layoutState = learnerMaster.getZxidLayoutState(); + boolean isPeerNewEpochZxid = layoutState.layoutFor(peerLastZxid).getCounterFromZxid(peerLastZxid) == 0; long queuedZxid = peerLastZxid; // as we look through proposals, this variable keeps track of previous // proposal Id. @@ -995,7 +1047,8 @@ protected long queueCommittedProposals(Iterator itr, long peerLastZxid } else if (packetZxid > peerLastZxid) { // Peer have some proposals that the learnerMaster hasn't seen yet // it may used to be a leader - if (ZxidUtils.getEpochFromZxid(packetZxid) != ZxidUtils.getEpochFromZxid(peerLastZxid)) { + if (layoutState.layoutFor(packetZxid).getEpochFromZxid(packetZxid) + != layoutState.layoutFor(peerLastZxid).getEpochFromZxid(peerLastZxid)) { // We cannot send TRUNC that cross epoch boundary. // The learner will crash if it is asked to do so. // We will send snapshot this those cases. diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LearnerMaster.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LearnerMaster.java index 9bf6032af68..0ce73ca5933 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LearnerMaster.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LearnerMaster.java @@ -24,6 +24,7 @@ import org.apache.zookeeper.server.Request; import org.apache.zookeeper.server.ZKDatabase; import org.apache.zookeeper.server.quorum.auth.QuorumAuthServer; +import org.apache.zookeeper.server.util.ZxidLayoutState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -150,6 +151,12 @@ public LearnerSyncThrottler getLearnerDiffSyncThrottler() { */ abstract long getLastProposed(); + /** + * the zxid layout state of the ensemble member backing this master + * @return the zxid layout state + */ + abstract ZxidLayoutState getZxidLayoutState(); + /** * the current tick * @return the current tick diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/ObserverMaster.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/ObserverMaster.java index 2369eabe36c..c7534cae014 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/ObserverMaster.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/ObserverMaster.java @@ -44,6 +44,7 @@ import org.apache.zookeeper.server.Request; import org.apache.zookeeper.server.ZKDatabase; import org.apache.zookeeper.server.quorum.auth.QuorumAuthServer; +import org.apache.zookeeper.server.util.ZxidLayoutState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -218,9 +219,14 @@ public int getCurrentTick() { return self.tick.get(); } + @Override + public ZxidLayoutState getZxidLayoutState() { + return self.getZxidLayoutState(); + } + @Override public void processAck(long sid, long zxid, SocketAddress localSocketAddress) { - if ((zxid & 0xffffffffL) == 0) { + if (self.getZxidLayoutState().current().getCounterFromZxid(zxid) == 0) { /* * We no longer process NEWLEADER ack by this method. However, * the learner sends ack back to the leader after it gets UPTODATE diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java index 56dcbd224d1..f277509bd2e 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java @@ -85,6 +85,8 @@ import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; import org.apache.zookeeper.server.util.ConfigUtils; import org.apache.zookeeper.server.util.JvmPauseMonitor; +import org.apache.zookeeper.server.util.ZxidLayout; +import org.apache.zookeeper.server.util.ZxidLayoutState; import org.apache.zookeeper.server.util.ZxidUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1205,11 +1207,21 @@ public synchronized void start() { private void loadDataBase() { try { + // Restore the zxid layout first: every zxid parsed below and by + // the rest of this member's lifetime depends on it. + try { + zxidLayoutState.switchAt(readLongFromFile(ZXID_LAYOUT_SWITCH_EPOCH_FILENAME)); + LOG.info("Using the {} zxid layout from epoch 0x{} on", + ZxidLayout.WIDE_COUNTER, Long.toHexString(zxidLayoutState.getSwitchEpoch())); + } catch (FileNotFoundException e) { + // never switched: everything is in the legacy layout + } + zkDb.setZxidLayoutState(zxidLayoutState); zkDb.loadDataBase(); // load the epochs long lastProcessedZxid = zkDb.getDataTree().lastProcessedZxid; - long epochOfZxid = ZxidUtils.getEpochFromZxid(lastProcessedZxid); + long epochOfZxid = zxidLayoutState.layoutFor(lastProcessedZxid).getEpochFromZxid(lastProcessedZxid); try { currentEpoch = readLongFromFile(CURRENT_EPOCH_FILENAME); } catch (FileNotFoundException e) { @@ -2311,6 +2323,105 @@ private long readLongFromFile(String name) throws IOException { public static final String ACCEPTED_EPOCH_FILENAME = "acceptedEpoch"; + public static final String ZXID_LAYOUT_SWITCH_EPOCH_FILENAME = "zxidLayoutSwitchEpoch"; + + /** + * Whether a newly elected leader may switch the ensemble to the + * wide-counter (24-bit epoch / 40-bit counter) zxid layout, see + * ZOOKEEPER-2789. Enable it only after every member of the ensemble + * runs a binary that understands the layout (learner protocol version + * 0x11000 or above): once a leader has switched, members that do not + * understand the layout are refused, and the switch cannot be undone. + */ + public static final String WIDE_COUNTER_ZXID_ENABLED = "zookeeper.wideCounterZxidEnabled"; + + private final boolean wideCounterZxidEnabled = Boolean.getBoolean(WIDE_COUNTER_ZXID_ENABLED); + + private final ZxidLayoutState zxidLayoutState = new ZxidLayoutState(); + + public boolean isWideCounterZxidEnabled() { + return wideCounterZxidEnabled; + } + + public ZxidLayoutState getZxidLayoutState() { + return zxidLayoutState; + } + + /** + * Records that the ensemble uses the wide-counter zxid layout from the + * given epoch on, both in memory and on disk, so a restart keeps parsing + * the on-disk data with the correct layouts. No-op if this switch epoch + * is already recorded. + * + * @throws IOException if the switch epoch cannot be persisted + * @throws IllegalStateException if a different switch epoch was already recorded + */ + public void recordZxidLayoutSwitch(long epoch) throws IOException { + if (zxidLayoutState.isSwitched() && zxidLayoutState.getSwitchEpoch() == epoch) { + return; + } + zxidLayoutState.switchAt(epoch); + writeLongToFile(ZXID_LAYOUT_SWITCH_EPOCH_FILENAME, epoch); + LOG.info("Switched to the {} zxid layout starting from epoch 0x{}", ZxidLayout.WIDE_COUNTER, Long.toHexString(epoch)); + } + + /** + * Adopts the zxid layout switch epoch announced by the leader during the + * LEADERINFO handshake. + * + *

A locally recorded switch epoch may only be moved while this member + * holds no wide-counter data yet, which happens when the leader that + * announced the previous switch epoch failed before its epoch committed + * anything. Once wide-counter zxids exist on disk, a different + * announcement means the histories diverged (e.g. an unsupported + * downgrade happened) and joining would corrupt the zxid order. + * + * @throws IOException if the epochs conflict or persisting fails + */ + public void adoptZxidLayoutSwitch(long epoch) throws IOException { + if (zxidLayoutState.isSwitched() && zxidLayoutState.getSwitchEpoch() == epoch) { + return; + } + if (zxidLayoutState.isSwitched()) { + checkZxidLayoutSwitchUnused("move the zxid layout switch epoch to 0x" + Long.toHexString(epoch)); + zxidLayoutState.forceSwitchAt(epoch); + } else { + zxidLayoutState.switchAt(epoch); + } + writeLongToFile(ZXID_LAYOUT_SWITCH_EPOCH_FILENAME, epoch); + LOG.info("Adopted the {} zxid layout switch at epoch 0x{} from the leader", + ZxidLayout.WIDE_COUNTER, Long.toHexString(epoch)); + } + + /** + * Called when the leader announces no zxid layout switch. A locally + * recorded but still unused switch is forgotten (the leader that + * announced it failed before its epoch committed anything, and the + * current leader does not have the feature enabled). A used one makes + * joining impossible, see {@link #adoptZxidLayoutSwitch(long)}. + */ + public void abandonUnusedZxidLayoutSwitch() throws IOException { + if (!zxidLayoutState.isSwitched()) { + return; + } + checkZxidLayoutSwitchUnused("abandon the zxid layout switch"); + zxidLayoutState.clear(); + File file = new File(logFactory.getSnapDir(), ZXID_LAYOUT_SWITCH_EPOCH_FILENAME); + if (file.exists() && !file.delete()) { + throw new IOException("Failed to delete " + file); + } + LOG.info("Abandoned the unused {} zxid layout switch: the current leader did not announce one", ZxidLayout.WIDE_COUNTER); + } + + private void checkZxidLayoutSwitchUnused(String action) throws IOException { + long firstWideZxid = ZxidLayout.WIDE_COUNTER.makeZxid(zxidLayoutState.getSwitchEpoch(), 0); + if (Long.compareUnsigned(getLastLoggedZxid(), firstWideZxid) >= 0) { + throw new IOException("Refusing to " + action + ": this member already holds zxids in the " + + ZxidLayout.WIDE_COUNTER + " layout of epoch 0x" + Long.toHexString(zxidLayoutState.getSwitchEpoch()) + + ", the leader's history diverged from ours (was the ensemble downgraded?)"); + } + } + /** * Write a long value to disk atomically. Either succeeds or an exception * is thrown. diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/util/LogChopper.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/util/LogChopper.java index ccc87334aa7..d69750cb420 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/util/LogChopper.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/util/LogChopper.java @@ -68,7 +68,17 @@ public static void main(String[] args) { ServiceUtils.requestSystemExit(rc.getValue()); } + /** + * Chops with legacy-layout gap diagnostics. Callers that know the log + * spans a wide-counter layout switch should use + * {@link #chop(InputStream, OutputStream, long, ZxidLayoutState)} so the + * diagnostics decompose zxids with the correct layout. + */ public static boolean chop(InputStream is, OutputStream os, long zxid) throws IOException { + return chop(is, os, zxid, ZxidLayoutState.legacyOnly()); + } + + public static boolean chop(InputStream is, OutputStream os, long zxid, ZxidLayoutState layoutState) throws IOException { BinaryInputArchive logStream = BinaryInputArchive.getArchive(is); BinaryOutputArchive choppedStream = BinaryOutputArchive.getArchive(os); FileHeader fhdr = new FileHeader(); @@ -127,9 +137,10 @@ public static boolean chop(InputStream is, OutputStream os, long zxid) throws IO // logging the gap to make the inconsistency investigation easier if (previousZxid != -1 && txnZxid != previousZxid + 1) { - long txnEpoch = ZxidUtils.getEpochFromZxid(txnZxid); - long txnCounter = ZxidUtils.getCounterFromZxid(txnZxid); - long previousEpoch = ZxidUtils.getEpochFromZxid(previousZxid); + ZxidLayout txnLayout = layoutState.layoutFor(txnZxid); + long txnEpoch = txnLayout.getEpochFromZxid(txnZxid); + long txnCounter = txnLayout.getCounterFromZxid(txnZxid); + long previousEpoch = layoutState.layoutFor(previousZxid).getEpochFromZxid(previousZxid); if (txnEpoch == previousEpoch) { System.out.println(String.format("There is intra-epoch gap between %x and %x", previousZxid, txnZxid)); } else if (txnCounter != 1) { diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/util/ZxidLayout.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/util/ZxidLayout.java new file mode 100644 index 00000000000..bacae238d36 --- /dev/null +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/util/ZxidLayout.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.util; + +/** + * A bit layout of a zxid: how the 64 bits are split between the epoch part + * (high bits) and the counter part (low bits). + * + *

Two layouts exist: + *

    + *
  • {@link #LEGACY}: 32-bit epoch / 32-bit counter, the layout ZooKeeper + * has always used and the default.
    • + *
  • {@link #WIDE_COUNTER}: 24-bit epoch / 40-bit counter. With a 32-bit + * counter, a cluster sustaining 1k writes/s exhausts the counter in about + * 49.7 days and every exhaustion forces a leader re-election (see + * ZOOKEEPER-1277 and ZOOKEEPER-2789). With a 40-bit counter the same load + * lasts about 34.9 years, while 24 bits of epoch still allow one leader + * election per hour for roughly 1915 years.
    • + *
+ * + *

Which layout applies to a given zxid is decided by + * {@link ZxidLayoutState}: an ensemble starts with {@link #LEGACY} and may + * switch to {@link #WIDE_COUNTER} from a specific epoch on. Instances are + * immutable; the static {@link ZxidUtils} helpers are equivalent to + * {@link #LEGACY} and remain for the fixed-layout uses (such as the epoch + * carrier zxids of the FOLLOWERINFO / LEADERINFO handshake packets, which + * stay in the legacy layout on the wire regardless of the data layout). + * + *

Introduced by Benedict Jin (asdf2014) for ZOOKEEPER-2789. + */ +public final class ZxidLayout { + + public static final ZxidLayout LEGACY = new ZxidLayout("legacy", 32); + public static final ZxidLayout WIDE_COUNTER = new ZxidLayout("wide-counter", 40); + + private final String name; + private final int counterBits; + private final long counterMask; + private final long maxEpoch; + + private ZxidLayout(String name, int counterBits) { + this.name = name; + this.counterBits = counterBits; + this.counterMask = (1L << counterBits) - 1; + // The largest epoch that keeps a zxid non-negative, so that numeric + // (signed) zxid comparisons all over the code base stay valid. + this.maxEpoch = (1L << (63 - counterBits)) - 1; + } + + public long getEpochFromZxid(long zxid) { + return zxid >> counterBits; + } + + public long getCounterFromZxid(long zxid) { + return zxid & counterMask; + } + + public long makeZxid(long epoch, long counter) { + return (epoch << counterBits) | (counter & counterMask); + } + + /** Returns the given zxid with its counter part zeroed out, keeping only the epoch part. */ + public long clearCounter(long zxid) { + return zxid & ~counterMask; + } + + /** Returns the maximum value the counter part of a zxid can hold before it rolls over. */ + public long getMaxCounter() { + return counterMask; + } + + /** Returns the maximum epoch this layout can hold without producing a negative zxid. */ + public long getMaxEpoch() { + return maxEpoch; + } + + public int getCounterBits() { + return counterBits; + } + + @Override + public String toString() { + return name; + } + +} diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/util/ZxidLayoutState.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/util/ZxidLayoutState.java new file mode 100644 index 00000000000..6ef352ef60e --- /dev/null +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/util/ZxidLayoutState.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.util; + +/** + * Tracks which {@link ZxidLayout} an ensemble member uses, and from which + * epoch on. + * + *

An ensemble starts in the {@link ZxidLayout#LEGACY} layout. When the + * wide-counter layout is enabled (see {@code zookeeper.wideCounterZxidEnabled}) + * and a new leader is elected, the leader switches the ensemble to + * {@link ZxidLayout#WIDE_COUNTER} starting from its own new epoch (the + * "switch epoch"), and announces that epoch to the learners during the + * LEADERINFO handshake. Every member persists the switch epoch next to + * {@code currentEpoch}, so that after a restart it can still parse its + * on-disk data correctly. + * + *

Because the switch always happens together with an epoch bump, the + * numeric order of zxids is preserved across the switch: the first + * wide-counter zxid {@code (switchEpoch << 40)} is numerically larger than + * every legacy zxid of the previous epochs. This has two consequences: + *

    + *
  • Pure numeric zxid comparisons (leader election, DIFF/TRUNC ranges, + * min/max committed log) remain valid without any layout knowledge.
    • + *
  • For the places that need to decompose a zxid into epoch and + * counter, {@link #layoutFor(long)} picks the correct layout for any zxid, + * old or new, by comparing it against the first wide-counter zxid.
    • + *
+ * + *

The switch is one-way: once a member has recorded a switch epoch it + * must never go back to the legacy layout (an old-layout leader elected + * afterwards would generate zxids that sort below already-committed ones). + * {@link #adopt(long)} therefore rejects any attempt to move or clear the + * switch epoch. + * + *

Introduced by Benedict Jin (asdf2014) for ZOOKEEPER-2789. + */ +public class ZxidLayoutState { + + /** Sentinel: the ensemble has not switched, everything is in the legacy layout. */ + public static final long NOT_SWITCHED = Long.MAX_VALUE; + + private static final ZxidLayoutState LEGACY_ONLY = new ZxidLayoutState() { + @Override + public void switchAt(long epoch) { + throw new UnsupportedOperationException("this ZxidLayoutState is immutable"); + } + @Override + public void forceSwitchAt(long epoch) { + throw new UnsupportedOperationException("this ZxidLayoutState is immutable"); + } + @Override + public void clear() { + throw new UnsupportedOperationException("this ZxidLayoutState is immutable"); + } + }; + + private volatile long switchEpoch = NOT_SWITCHED; + + /** + * Returns a shared immutable state that never switches. Used as the + * default wherever no quorum context exists (e.g. standalone servers), + * keeping the legacy behavior. + */ + public static ZxidLayoutState legacyOnly() { + return LEGACY_ONLY; + } + + public boolean isSwitched() { + return switchEpoch != NOT_SWITCHED; + } + + public long getSwitchEpoch() { + return switchEpoch; + } + + /** The layout for newly generated zxids. */ + public ZxidLayout current() { + return isSwitched() ? ZxidLayout.WIDE_COUNTER : ZxidLayout.LEGACY; + } + + /** + * The layout the given zxid was generated with. Correct for any zxid, + * old or new: zxids from epochs before the switch epoch are numerically + * smaller than the first wide-counter zxid. + */ + public ZxidLayout layoutFor(long zxid) { + long epoch = switchEpoch; + if (epoch == NOT_SWITCHED) { + return ZxidLayout.LEGACY; + } + long firstWideZxid = ZxidLayout.WIDE_COUNTER.makeZxid(epoch, 0); + return Long.compareUnsigned(zxid, firstWideZxid) >= 0 ? ZxidLayout.WIDE_COUNTER : ZxidLayout.LEGACY; + } + + /** + * Switches to the wide-counter layout starting from the given epoch. + * Called by the leader when it establishes a new epoch with the feature + * enabled, and when loading a previously persisted switch epoch. + * + * @throws IllegalStateException if already switched at a different epoch + * @throws IllegalArgumentException if the epoch cannot be represented + */ + public void switchAt(long epoch) { + checkRange(epoch); + synchronized (this) { + if (isSwitched() && switchEpoch != epoch) { + throw new IllegalStateException( + "already switched to " + ZxidLayout.WIDE_COUNTER + " at epoch " + switchEpoch + ", cannot re-switch at " + epoch); + } + switchEpoch = epoch; + } + } + + /** + * Moves the switch epoch unconditionally. Only safe while no zxid has + * been generated in the wide-counter layout yet — e.g. when the leader + * that announced the previous switch epoch failed before its epoch + * committed anything, and the next leader announces a different one. + * Callers must verify that (see QuorumPeer#adoptZxidLayoutSwitch). + */ + public void forceSwitchAt(long epoch) { + checkRange(epoch); + synchronized (this) { + switchEpoch = epoch; + } + } + + /** + * Forgets a recorded switch, returning to the legacy-only state. Like + * {@link #forceSwitchAt(long)} this is only safe while no wide-counter + * zxid exists yet. + */ + public void clear() { + synchronized (this) { + switchEpoch = NOT_SWITCHED; + } + } + + private static void checkRange(long epoch) { + if (epoch < 0 || epoch > ZxidLayout.WIDE_COUNTER.getMaxEpoch()) { + throw new IllegalArgumentException("switch epoch " + epoch + " out of range for " + ZxidLayout.WIDE_COUNTER); + } + } + +} diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZxidRolloverTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZxidRolloverTest.java index 495c1507e43..eff75561d5a 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZxidRolloverTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZxidRolloverTest.java @@ -28,6 +28,7 @@ import org.apache.zookeeper.ZKTestCase; import org.apache.zookeeper.ZooDefs.Ids; import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.server.util.ZxidLayout; import org.apache.zookeeper.test.ClientBase; import org.apache.zookeeper.test.ClientBase.CountdownWatcher; import org.apache.zookeeper.test.ClientTest; @@ -211,7 +212,9 @@ private void shutdown(int idx) throws Exception { /** Reset the next zxid to be near epoch end */ private void adjustEpochNearEnd() { - zksLeader.setZxid((zksLeader.getZxid() & 0xffffffff00000000L) | 0xfffffffcL); + // This test never enables the wide-counter layout, so the zxid stays + // in the legacy 32/32 layout throughout. + zksLeader.setZxid(ZxidLayout.LEGACY.clearCounter(zksLeader.getZxid()) | (ZxidLayout.LEGACY.getMaxCounter() - 3)); } @AfterEach diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/persistence/TxnLogToolkitTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/persistence/TxnLogToolkitTest.java index 361666709f4..979e813282c 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/persistence/TxnLogToolkitTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/persistence/TxnLogToolkitTest.java @@ -27,6 +27,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintStream; import java.nio.ByteBuffer; @@ -36,11 +37,14 @@ import java.util.Scanner; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.zip.Adler32; +import java.util.zip.Checksum; import org.apache.commons.io.FileUtils; import org.apache.jute.BinaryOutputArchive; import org.apache.jute.Record; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.server.util.ZxidLayout; import org.apache.zookeeper.test.ClientBase; import org.apache.zookeeper.txn.CheckVersionTxn; import org.apache.zookeeper.txn.CreateContainerTxn; @@ -51,6 +55,7 @@ import org.apache.zookeeper.txn.MultiTxn; import org.apache.zookeeper.txn.SetDataTxn; import org.apache.zookeeper.txn.Txn; +import org.apache.zookeeper.txn.TxnHeader; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -84,10 +89,10 @@ public void tearDown() throws IOException { public void testDumpMode() throws Exception { // Arrange File logfile = new File(new File(mySnapDir, "version-2"), "log.274"); - TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true); - - // Act - lt.dump(null); + try (TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true)) { + // Act + lt.dump(null); + } // Assert // no exception thrown @@ -146,10 +151,10 @@ public void testInitWithRecoveryFileExists() { public void testDumpWithCrcError() throws Exception { // Arrange File logfile = new File(new File(mySnapDir, "version-2"), "log.42"); - TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true); - - // Act - lt.dump(null); + try (TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true)) { + // Act + lt.dump(null); + } // Assert String output = outContent.toString(); @@ -162,10 +167,10 @@ public void testDumpWithCrcError() throws Exception { public void testRecoveryFixBrokenFile() throws Exception { // Arrange File logfile = new File(new File(mySnapDir, "version-2"), "log.42"); - TxnLogToolkit lt = new TxnLogToolkit(true, false, logfile.toString(), true); - - // Act - lt.dump(null); + try (TxnLogToolkit lt = new TxnLogToolkit(true, false, logfile.toString(), true)) { + // Act + lt.dump(null); + } // Assert String output = outContent.toString(); @@ -174,8 +179,9 @@ public void testRecoveryFixBrokenFile() throws Exception { // Should be able to dump the recovered logfile with no CRC error outContent.reset(); logfile = new File(new File(mySnapDir, "version-2"), "log.42.fixed"); - lt = new TxnLogToolkit(false, false, logfile.toString(), true); - lt.dump(null); + try (TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true)) { + lt.dump(null); + } output = outContent.toString(); assertThat(output, not(containsString("CRC ERROR"))); } @@ -184,10 +190,10 @@ public void testRecoveryFixBrokenFile() throws Exception { public void testRecoveryInteractiveMode() throws Exception { // Arrange File logfile = new File(new File(mySnapDir, "version-2"), "log.42"); - TxnLogToolkit lt = new TxnLogToolkit(true, false, logfile.toString(), false); - - // Act - lt.dump(new Scanner("y\n")); + try (TxnLogToolkit lt = new TxnLogToolkit(true, false, logfile.toString(), false)) { + // Act + lt.dump(new Scanner("y\n")); + } // Assert String output = outContent.toString(); @@ -196,10 +202,59 @@ public void testRecoveryInteractiveMode() throws Exception { // Should be able to dump the recovered logfile with no CRC error outContent.reset(); logfile = new File(new File(mySnapDir, "version-2"), "log.42.fixed"); - lt = new TxnLogToolkit(false, false, logfile.toString(), true); - lt.dump(null); + try (TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true)) { + lt.dump(null); + } output = outContent.toString(); assertThat(output, not(containsString("CRC ERROR"))); } + /** + * Chopping a txnlog that spans the legacy -> wide-counter layout boundary: + * the chop result is layout-independent (the boundary is a numeric + * comparison), but the gap diagnostics decompose zxids into epoch/counter, + * so they need the switch epoch to classify a gap correctly. Without it a + * within-epoch wide-counter gap is misread as an inter-epoch gap. + */ + @Test + public void testChopWithWideCounterSwitchEpoch() throws Exception { + // Two wide-counter zxids of epoch 6, straddling a 32-bit boundary, with + // an intra-epoch gap between them. Legacy 32/32 math would read them as + // epochs 1536 and 1537 (an inter-epoch gap); wide 24/40 math reads both + // as epoch 6 (an intra-epoch gap). + long wideA = ZxidLayout.WIDE_COUNTER.makeZxid(6, 0xFF000000L); + long wideB = ZxidLayout.WIDE_COUNTER.makeZxid(6, 0x100000005L); + File logFile = new File(mySnapDir, "log.wide"); + writeTxnLog(logFile, wideA, wideB); + + // With the switch epoch, the gap is correctly classified intra-epoch. + outContent.reset(); + new TxnLogToolkit(logFile.toString(), "0x" + Long.toHexString(wideA), "6").chop(); + String withEpoch = outContent.toString(); + assertThat(withEpoch, containsString("intra-epoch gap")); + assertThat(withEpoch, not(containsString("inter-epoch gap"))); + assertTrue(new File(mySnapDir, "log.wide.chopped" + wideA).exists()); + + // Without it (legacy-only), the same gap is misreported as inter-epoch. + outContent.reset(); + new TxnLogToolkit(logFile.toString(), "0x" + Long.toHexString(wideA)).chop(); + assertThat(outContent.toString(), containsString("inter-epoch gap")); + } + + private void writeTxnLog(File logFile, long... zxids) throws IOException { + try (FileOutputStream fos = new FileOutputStream(logFile)) { + BinaryOutputArchive oa = BinaryOutputArchive.getArchive(fos); + new FileHeader(FileTxnLog.TXNLOG_MAGIC, 2, 1).serialize(oa, "fileheader"); + for (long zxid : zxids) { + TxnHeader hdr = new TxnHeader(1, 1, zxid, 1, ZooDefs.OpCode.error); + byte[] bytes = Util.marshallTxnEntry(hdr, new ErrorTxn(1), null); + Checksum crc = new Adler32(); + crc.update(bytes, 0, bytes.length); + oa.writeLong(crc.getValue(), "crcvalue"); + oa.writeBuffer(bytes, "txnEntry"); + oa.writeByte((byte) 'B', "EOR"); + } + } + } + } diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/FLEMalformedNotificationMessageTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/FLEMalformedNotificationMessageTest.java index 30187644a87..b6dfd14a6ed 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/FLEMalformedNotificationMessageTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/FLEMalformedNotificationMessageTest.java @@ -18,16 +18,19 @@ package org.apache.zookeeper.server.quorum; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.fail; import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.HashMap; +import java.util.concurrent.TimeUnit; import org.apache.zookeeper.PortAssignment; import org.apache.zookeeper.ZKTestCase; import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer; import org.apache.zookeeper.server.quorum.QuorumPeer.ServerState; +import org.apache.zookeeper.server.util.ZxidUtils; import org.apache.zookeeper.test.ClientBase; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -113,6 +116,44 @@ public void testTooShortPartialNotificationMessage() throws Exception { } + @Test + public void testLegacy28ByteNotificationMessage() throws Exception { + + /* + * Start mock server 1, send a notification in the 28-byte format of + * releases before 3.4.6: state + leader + zxid + electionEpoch, with + * no peer epoch and no version. The receiver extrapolates the peer + * epoch from the zxid, decomposing it with the layout the zxid was + * generated with. + */ + startMockServer(1); + // Wait for the election peer's own vote to reach us first: by then + // the connection pair is fully established. A message sent earlier + // can be swallowed while the initial connection is torn down and + // re-established the other way around, and this test needs the + // 28-byte vote to actually reach the receiver. + assertNotNull(mockCnxManager.pollRecvQueue(5000, TimeUnit.MILLISECONDS)); + byte[] requestBytes = new byte[28]; + ByteBuffer requestBuffer = ByteBuffer.wrap(requestBytes); + requestBuffer.clear(); + requestBuffer.putInt(ServerState.LOOKING.ordinal()); // state + requestBuffer.putLong(0); // leader + requestBuffer.putLong(ZxidUtils.makeZxid(1, 42)); // zxid, carries the epoch + requestBuffer.putLong(0); // electionEpoch + mockCnxManager.toSend(0L, requestBuffer); + + /* + * Assert that the message receiver thread in leader election is still healthy: + * we are sending valid votes and waiting for the leader election to be finished. + */ + sendValidNotifications(1, 0); + leaderElectionThread.join(5000); + if (leaderElectionThread.isAlive()) { + fail("Leader election thread didn't join, something went wrong."); + } + } + + @Test public void testNotificationMessageWithNegativeConfigLength() throws Exception { diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/LearnerHandlerTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/LearnerHandlerTest.java index e38c82ea9f7..d6fb582c7d5 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/LearnerHandlerTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/LearnerHandlerTest.java @@ -37,6 +37,8 @@ import org.apache.zookeeper.server.ZKDatabase; import org.apache.zookeeper.server.persistence.FileTxnSnapLog; import org.apache.zookeeper.server.quorum.Leader.Proposal; +import org.apache.zookeeper.server.util.ZxidLayout; +import org.apache.zookeeper.server.util.ZxidLayoutState; import org.apache.zookeeper.server.util.ZxidUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -145,6 +147,7 @@ public Long answer(InvocationOnMock invocation) { } }); when(leader.getZKDatabase()).thenReturn(db); + when(leader.getZxidLayoutState()).thenReturn(new ZxidLayoutState()); learnerHandler = new MockLearnerHandler(sock, leader); } @@ -338,6 +341,7 @@ public Iterator getProposalsFromTxnLog(long peerZxid, long limit) { db.txnLog.add(createProposal(2)); db.txnLog.add(createProposal(3)); when(leader.getZKDatabase()).thenReturn(db); + when(leader.getZxidLayoutState()).thenReturn(new ZxidLayoutState()); // Peer zxid peerZxid = 4; @@ -394,6 +398,65 @@ long getZxid(long epoch, long counter) { return ZxidUtils.makeZxid(epoch, counter); } + /** + * Test the sync decisions when the ensemble has switched to the + * wide-counter zxid layout (ZOOKEEPER-2789) while the data — and the + * last zxid the follower hands in — still spans the legacy epochs. + * Zxids below the switch epoch must be decomposed with the legacy + * layout: parsed with the wide layout, a legacy new-epoch zxid like + * (5, 0) would read as having a non-zero counter and the sync logic + * would take the wrong branch. + */ + @Test + public void testSyncAcrossZxidLayoutSwitch() throws Exception { + // The ensemble switched to the wide-counter layout at epoch 6. + ZxidLayoutState layoutState = new ZxidLayoutState(); + layoutState.switchAt(6); + when(leader.getZxidLayoutState()).thenReturn(layoutState); + + long legacy41 = getZxid(4, 1); + long legacy51 = getZxid(5, 1); + long legacy52 = getZxid(5, 2); + long wide61 = ZxidLayout.WIDE_COUNTER.makeZxid(6, 1); + + db.txnLog.add(createProposal(legacy41)); + db.txnLog.add(createProposal(legacy51)); + db.txnLog.add(createProposal(legacy52)); + db.txnLog.add(createProposal(wide61)); + db.committedLog.add(createProposal(legacy51)); + db.committedLog.add(createProposal(legacy52)); + db.committedLog.add(createProposal(wide61)); + db.lastProcessedZxid = wide61; + + // A follower at a legacy new-epoch zxid (epoch 5, counter 0): the + // legacy decomposition marks it a new-epoch zxid, TRUNC logic is + // skipped, and it catches up with a DIFF across the layout boundary. + long peerZxid = getZxid(5, 0); + assertFalse(learnerHandler.syncFollower(peerZxid, leader)); + assertOpType(Leader.DIFF, wide61, wide61); + // DIFF + 3 proposals + 3 commits + assertEquals(7, learnerHandler.getQueuedPackets().size()); + queuedPacketMatches(new long[]{legacy51, legacy52, wide61}); + reset(); + + // A follower already at the last legacy zxid only syncs the + // wide-counter commits. + peerZxid = legacy52; + assertFalse(learnerHandler.syncFollower(peerZxid, leader)); + assertOpType(Leader.DIFF, wide61, wide61); + // DIFF + 1 proposal + 1 commit + assertEquals(3, learnerHandler.getQueuedPackets().size()); + queuedPacketMatches(new long[]{wide61}); + reset(); + + // A follower already in the wide layout is in sync. + peerZxid = wide61; + assertFalse(learnerHandler.syncFollower(peerZxid, leader)); + assertOpType(Leader.DIFF, wide61, wide61); + assertEquals(1, learnerHandler.getQueuedPackets().size()); + reset(); + } + /** * Test cases with zxids that are negative long */ diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerZxidLayoutTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerZxidLayoutTest.java new file mode 100644 index 00000000000..9a6f166994a --- /dev/null +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerZxidLayoutTest.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.quorum; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.zookeeper.ZKTestCase; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.server.DataTree; +import org.apache.zookeeper.server.Request; +import org.apache.zookeeper.server.ZKDatabase; +import org.apache.zookeeper.server.persistence.FileTxnSnapLog; +import org.apache.zookeeper.server.util.ZxidLayout; +import org.apache.zookeeper.txn.ErrorTxn; +import org.apache.zookeeper.txn.TxnHeader; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests the per-QuorumPeer zxid layout switch bookkeeping introduced by + * ZOOKEEPER-2789: persisting a switch epoch, adopting one announced by a + * leader, and the one-way / anti-downgrade guards that refuse to move or + * forget a switch once wide-counter data has been written. + * + *

Introduced by Benedict Jin (asdf2014) for ZOOKEEPER-2789. + */ +public class QuorumPeerZxidLayoutTest extends ZKTestCase { + + private final List logFactories = new ArrayList<>(); + + @AfterEach + public void tearDown() throws IOException { + // release the txn log handles so the @TempDir can be deleted on Windows + for (FileTxnSnapLog logFactory : logFactories) { + logFactory.close(); + } + } + + /** + * A bare QuorumPeer whose last logged zxid is {@code lastLoggedZxid}, used + * to drive the "has this member already written wide-counter data?" guard. + */ + private QuorumPeer newPeer(File tmpDir, long lastLoggedZxid) throws Exception { + tmpDir.mkdirs(); + FileTxnSnapLog logFactory = new FileTxnSnapLog(tmpDir, tmpDir); + logFactories.add(logFactory); + logFactory.save(new DataTree(), new ConcurrentHashMap<>(), false); + if (lastLoggedZxid != 0) { + logFactory.append(new Request(1, 1, ZooDefs.OpCode.error, + new TxnHeader(1, 1, lastLoggedZxid, 1, ZooDefs.OpCode.error), new ErrorTxn(1), lastLoggedZxid)); + logFactory.commit(); + } + QuorumPeer peer = QuorumPeer.testingQuorumPeer(); + peer.setTxnFactory(logFactory); + peer.setZKDatabase(new ZKDatabase(logFactory)); + peer.getLastLoggedZxid(); // force the database to load + assertEquals(lastLoggedZxid, peer.getLastLoggedZxid()); + return peer; + } + + private File switchEpochFile(File tmpDir) { + // FileTxnSnapLog keeps the epoch files under the version-2 sub-directory. + return new File(new File(tmpDir, "version-2"), QuorumPeer.ZXID_LAYOUT_SWITCH_EPOCH_FILENAME); + } + + @Test + public void testRecordZxidLayoutSwitchPersistsAndIsIdempotent(@TempDir File tmpDir) throws Exception { + QuorumPeer peer = newPeer(tmpDir, 0); + assertFalse(peer.getZxidLayoutState().isSwitched()); + + peer.recordZxidLayoutSwitch(6); + assertTrue(peer.getZxidLayoutState().isSwitched()); + assertEquals(6, peer.getZxidLayoutState().getSwitchEpoch()); + assertTrue(switchEpochFile(tmpDir).exists()); + + // Recording the same switch epoch again is a no-op. + peer.recordZxidLayoutSwitch(6); + assertEquals(6, peer.getZxidLayoutState().getSwitchEpoch()); + } + + @Test + public void testAdoptSwitchFromLeaderWhenNotSwitched(@TempDir File tmpDir) throws Exception { + QuorumPeer peer = newPeer(tmpDir, 0); + peer.adoptZxidLayoutSwitch(6); + assertTrue(peer.getZxidLayoutState().isSwitched()); + assertEquals(6, peer.getZxidLayoutState().getSwitchEpoch()); + assertTrue(switchEpochFile(tmpDir).exists()); + // Adopting the same epoch again is a no-op. + peer.adoptZxidLayoutSwitch(6); + assertEquals(6, peer.getZxidLayoutState().getSwitchEpoch()); + } + + @Test + public void testAdoptMovesUnusedSwitchEpoch(@TempDir File tmpDir) throws Exception { + // No wide-counter data yet, so a switch epoch that never committed + // anything may be moved when the next leader announces a different one + // (the previous announcing leader failed before its epoch committed). + QuorumPeer peer = newPeer(tmpDir, 0); + peer.recordZxidLayoutSwitch(6); + peer.adoptZxidLayoutSwitch(7); + assertEquals(7, peer.getZxidLayoutState().getSwitchEpoch()); + } + + @Test + public void testAdoptRefusesToMoveUsedSwitchEpoch(@TempDir File tmpDir) throws Exception { + // This member already holds a zxid in the wide-counter layout of epoch + // 6, so the switch is "used": a leader announcing a different switch + // epoch means histories diverged (an unsupported downgrade), and + // joining would corrupt the zxid order — it must be refused. + long wideZxid = ZxidLayout.WIDE_COUNTER.makeZxid(6, 100); + QuorumPeer peer = newPeer(tmpDir, wideZxid); + peer.recordZxidLayoutSwitch(6); + assertThrows(IOException.class, () -> peer.adoptZxidLayoutSwitch(7)); + // The recorded switch epoch is unchanged. + assertEquals(6, peer.getZxidLayoutState().getSwitchEpoch()); + } + + @Test + public void testAbandonUnusedSwitch(@TempDir File tmpDir) throws Exception { + QuorumPeer peer = newPeer(tmpDir, 0); + peer.recordZxidLayoutSwitch(6); + assertTrue(switchEpochFile(tmpDir).exists()); + + // The leader announced no switch and this member never wrote any + // wide-counter data, so the unused switch is forgotten. + peer.abandonUnusedZxidLayoutSwitch(); + assertFalse(peer.getZxidLayoutState().isSwitched()); + assertFalse(switchEpochFile(tmpDir).exists()); + } + + @Test + public void testAbandonIsNoOpWhenNotSwitched(@TempDir File tmpDir) throws Exception { + QuorumPeer peer = newPeer(tmpDir, 0); + peer.abandonUnusedZxidLayoutSwitch(); + assertFalse(peer.getZxidLayoutState().isSwitched()); + } + + @Test + public void testAbandonRefusesToForgetUsedSwitch(@TempDir File tmpDir) throws Exception { + // Wide-counter data exists, so the switch cannot be forgotten: doing so + // and then following a legacy-layout leader would break the zxid order. + long wideZxid = ZxidLayout.WIDE_COUNTER.makeZxid(6, 100); + QuorumPeer peer = newPeer(tmpDir, wideZxid); + peer.recordZxidLayoutSwitch(6); + assertThrows(IOException.class, peer::abandonUnusedZxidLayoutSwitch); + assertTrue(peer.getZxidLayoutState().isSwitched()); + } + +} diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/WideCounterZxidUpgradeTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/WideCounterZxidUpgradeTest.java new file mode 100644 index 00000000000..c2f73a7625b --- /dev/null +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/WideCounterZxidUpgradeTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.quorum; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.HashMap; +import java.util.Map; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.ZKTestCase; +import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.Stat; +import org.apache.zookeeper.server.admin.Commands; +import org.apache.zookeeper.server.util.ZxidLayout; +import org.apache.zookeeper.test.ClientBase; +import org.apache.zookeeper.test.QuorumUtil; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +/** + * End-to-end test of the upgrade to the wide-counter zxid layout + * (ZOOKEEPER-2789): an ensemble full of legacy-layout data is restarted + * with {@code zookeeper.wideCounterZxidEnabled}, the new leader switches + * the layout at its new epoch, the legacy data is synced correctly across + * the layout boundary, and the switch survives further restarts. + * + *

Introduced by Benedict Jin (asdf2014) for ZOOKEEPER-2789. + */ +public class WideCounterZxidUpgradeTest extends ZKTestCase { + + private final QuorumUtil qu = new QuorumUtil(1); // 3 servers + + @AfterEach + public void tearDown() throws Exception { + System.clearProperty(QuorumPeer.WIDE_COUNTER_ZXID_ENABLED); + qu.shutdownAll(); + qu.tearDown(); + } + + @Test + public void testUpgradeToWideCounterZxid() throws Exception { + qu.disableJMXTest = true; + + // Phase 1: a legacy ensemble commits some data. + qu.startAll(); + long legacyZxid; + ZooKeeper zk = ClientBase.createZKClient(qu.getConnString()); + try { + for (int i = 0; i < 10; i++) { + zk.create("/legacy-" + i, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + } + Stat stat = zk.exists("/legacy-9", false); + legacyZxid = stat.getCzxid(); + assertFalse(qu.getLeaderQuorumPeer().getZxidLayoutState().isSwitched()); + } finally { + zk.close(); + } + qu.shutdownAll(); + + // Phase 2: every binary now understands the wide layout and the + // feature is enabled; the newly elected leader switches the layout. + // The followers hand their legacy last zxids to the leader during + // the sync, exercising the mixed-layout parsing. + System.setProperty(QuorumPeer.WIDE_COUNTER_ZXID_ENABLED, "true"); + qu.startAll(); + long switchEpoch; + long wideZxid; + zk = ClientBase.createZKClient(qu.getConnString()); + try { + for (int i = 0; i < 10; i++) { + assertNotNull(zk.exists("/legacy-" + i, false), "legacy data lost across the layout switch: /legacy-" + i); + } + QuorumPeer leader = qu.getLeaderQuorumPeer(); + assertTrue(leader.getZxidLayoutState().isSwitched(), "the new leader should have switched the layout"); + switchEpoch = leader.getZxidLayoutState().getSwitchEpoch(); + + zk.create("/wide-0", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + Stat stat = zk.exists("/wide-0", false); + wideZxid = stat.getCzxid(); + assertEquals(switchEpoch, ZxidLayout.WIDE_COUNTER.getEpochFromZxid(wideZxid)); + assertTrue(wideZxid > legacyZxid, "the total zxid order must survive the layout switch"); + + // The operator-visible zabstate command decomposes the last zxid + // with the switched layout, so it reports the real 24-bit epoch + // rather than a legacy 32-bit misreading. + Map zabState = Commands.runGetCommand( + "zabstate", leader.getActiveServer(), new HashMap<>(), null, null).toMap(); + assertEquals(switchEpoch, ((Number) zabState.get("zab_epoch")).longValue()); + } finally { + zk.close(); + } + + // A follower that misses some wide-counter commits catches up with + // a sync over the mixed-layout data it holds. + int followerId = qu.getLeaderServer() == 1 ? 2 : 1; + qu.shutdown(followerId); + zk = ClientBase.createZKClient(qu.getConnString()); + try { + zk.create("/wide-1", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + } finally { + zk.close(); + } + qu.restart(followerId); + zk = ClientBase.createZKClient(qu.getConnectionStringForServer(followerId)); + try { + assertNotNull(zk.exists("/wide-1", false), "the restarted follower should have synced the wide-counter commits"); + assertNotNull(zk.exists("/legacy-0", false)); + } finally { + zk.close(); + } + + // Phase 3: the switch epoch is reloaded from disk after a full + // restart, and the ensemble keeps working in the wide layout. + qu.shutdownAll(); + qu.startAll(); + zk = ClientBase.createZKClient(qu.getConnString()); + try { + assertNotNull(zk.exists("/legacy-0", false)); + assertNotNull(zk.exists("/wide-0", false)); + QuorumPeer leader = qu.getLeaderQuorumPeer(); + assertTrue(leader.getZxidLayoutState().isSwitched()); + assertEquals(switchEpoch, leader.getZxidLayoutState().getSwitchEpoch(), + "the switch epoch must not move once recorded"); + zk.create("/wide-2", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + Stat stat = zk.exists("/wide-2", false); + // The restart elected a new epoch, still in the wide layout. + assertTrue(ZxidLayout.WIDE_COUNTER.getEpochFromZxid(stat.getCzxid()) > switchEpoch); + assertTrue(stat.getCzxid() > wideZxid); + } finally { + zk.close(); + } + } + +} diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/Zab1_0Test.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/Zab1_0Test.java index d374062e293..5bc44cb682a 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/Zab1_0Test.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/Zab1_0Test.java @@ -59,6 +59,7 @@ import org.apache.zookeeper.server.ZKDatabase; import org.apache.zookeeper.server.persistence.FileTxnSnapLog; import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer; +import org.apache.zookeeper.server.util.ZxidLayout; import org.apache.zookeeper.server.util.ZxidUtils; import org.apache.zookeeper.test.ClientBase; import org.apache.zookeeper.test.TestUtils; @@ -524,7 +525,7 @@ public void converseWithLeader(InputArchive ia, OutputArchive oa, Leader l, long readPacketSkippingPing(ia, qp); assertEquals(Leader.LEADERINFO, qp.getType()); assertEquals(ZxidUtils.makeZxid(2, 0), qp.getZxid()); - assertEquals(ByteBuffer.wrap(qp.getData()).getInt(), 0x10000); + assertEquals(ByteBuffer.wrap(qp.getData()).getInt(), LearnerHandler.WIDE_COUNTER_PROTOCOL_VERSION); assertEquals(2, l.self.getAcceptedEpoch()); assertEquals(1, l.self.getCurrentEpoch()); @@ -594,7 +595,7 @@ public void converseWithFollower(InputArchive ia, OutputArchive oa, Follower f) assertEquals(qp.getZxid(), 0); LearnerInfo learnInfo = new LearnerInfo(); ByteBufferInputStream.byteBuffer2Record(ByteBuffer.wrap(qp.getData()), learnInfo); - assertEquals(learnInfo.getProtocolVersion(), 0x10000); + assertEquals(learnInfo.getProtocolVersion(), LearnerHandler.WIDE_COUNTER_PROTOCOL_VERSION); assertEquals(learnInfo.getServerid(), 0); // We are simulating an established leader, so the epoch is 1 @@ -728,7 +729,7 @@ public void converseWithFollower(InputArchive ia, OutputArchive oa, Follower f) assertEquals(qp.getZxid(), 0); LearnerInfo learnInfo = new LearnerInfo(); ByteBufferInputStream.byteBuffer2Record(ByteBuffer.wrap(qp.getData()), learnInfo); - assertEquals(learnInfo.getProtocolVersion(), 0x10000); + assertEquals(learnInfo.getProtocolVersion(), LearnerHandler.WIDE_COUNTER_PROTOCOL_VERSION); assertEquals(learnInfo.getServerid(), 0); // We are simulating an established leader, so the epoch is 1 @@ -835,7 +836,7 @@ public void converseWithFollower(InputArchive ia, OutputArchive oa, Follower f) assertEquals(qp.getZxid(), 0); LearnerInfo learnInfo = new LearnerInfo(); ByteBufferInputStream.byteBuffer2Record(ByteBuffer.wrap(qp.getData()), learnInfo); - assertEquals(learnInfo.getProtocolVersion(), 0x10000); + assertEquals(learnInfo.getProtocolVersion(), LearnerHandler.WIDE_COUNTER_PROTOCOL_VERSION); assertEquals(learnInfo.getServerid(), 0); // We are simulating an established leader, so the epoch is 1 @@ -959,7 +960,7 @@ public void converseWithLeader(InputArchive ia, OutputArchive oa, Leader l) thro readPacketSkippingPing(ia, qp); assertEquals(Leader.LEADERINFO, qp.getType()); assertEquals(ZxidUtils.makeZxid(1, 0), qp.getZxid()); - assertEquals(ByteBuffer.wrap(qp.getData()).getInt(), 0x10000); + assertEquals(ByteBuffer.wrap(qp.getData()).getInt(), LearnerHandler.WIDE_COUNTER_PROTOCOL_VERSION); assertEquals(1, l.self.getAcceptedEpoch()); assertEquals(0, l.self.getCurrentEpoch()); @@ -984,6 +985,162 @@ public void converseWithLeader(InputArchive ia, OutputArchive oa, Leader l) thro }, testData); } + @Test + public void testInitialZxidTestingOnlyOverride(@TempDir File testData) throws Exception { + // The zookeeper.testingonly.initialZxid property makes the new leader + // replace the counter part of its zxid, so QA can start near the + // counter rollover without issuing billions of writes first. + System.setProperty("zookeeper.testingonly.initialZxid", "1000"); + try { + testLeaderConversation(new LeaderConversation() { + public void converseWithLeader(InputArchive ia, OutputArchive oa, Leader l) throws IOException, InterruptedException { + /* same handshake as testNormalRun */ + LearnerInfo li = new LearnerInfo(1, 0x10000, 0); + byte[] liBytes = RequestRecord.fromRecord(li).readBytes(); + QuorumPacket qp = new QuorumPacket(Leader.FOLLOWERINFO, 0, liBytes, null); + oa.writeRecord(qp, null); + + readPacketSkippingPing(ia, qp); + assertEquals(Leader.LEADERINFO, qp.getType()); + assertEquals(ZxidUtils.makeZxid(1, 0), qp.getZxid()); + + qp = new QuorumPacket(Leader.ACKEPOCH, 0, new byte[4], null); + oa.writeRecord(qp, null); + + readPacketSkippingPing(ia, qp); + assertEquals(Leader.DIFF, qp.getType()); + + readPacketSkippingPing(ia, qp); + assertEquals(Leader.NEWLEADER, qp.getType()); + assertEquals(ZxidUtils.makeZxid(1, 0), qp.getZxid()); + assertCurrentEpochGotUpdated(1, l.self, ClientBase.CONNECTION_TIMEOUT); + + qp = new QuorumPacket(Leader.ACK, qp.getZxid(), null, null); + oa.writeRecord(qp, null); + + readPacketSkippingPing(ia, qp); + assertEquals(Leader.UPTODATE, qp.getType()); + + // lead() applies the override right after startZkServer(), + // possibly after the UPTODATE packet is queued, so poll + // briefly instead of asserting immediately. + long deadline = System.currentTimeMillis() + ClientBase.CONNECTION_TIMEOUT; + while (ZxidUtils.getCounterFromZxid(l.zk.getZxid()) != 1000L && System.currentTimeMillis() < deadline) { + Thread.sleep(20); + } + // The counter part is replaced while the epoch part is kept. + assertEquals(ZxidUtils.makeZxid(1, 1000), l.zk.getZxid()); + } + }, testData); + } finally { + System.clearProperty("zookeeper.testingonly.initialZxid"); + } + } + + @Test + public void testWideCounterZxidSwitchHandshake(@TempDir File testData) throws Exception { + System.setProperty(QuorumPeer.WIDE_COUNTER_ZXID_ENABLED, "true"); + try { + testLeaderConversation(new LeaderConversation() { + public void converseWithLeader(InputArchive ia, OutputArchive oa, Leader l) throws IOException { + LearnerInfo li = new LearnerInfo(1, LearnerHandler.WIDE_COUNTER_PROTOCOL_VERSION, 0); + byte[] liBytes = RequestRecord.fromRecord(li).readBytes(); + QuorumPacket qp = new QuorumPacket(Leader.FOLLOWERINFO, 0, liBytes, null); + oa.writeRecord(qp, null); + + readPacketSkippingPing(ia, qp); + assertEquals(Leader.LEADERINFO, qp.getType()); + // The LEADERINFO zxid stays an epoch carrier in the + // legacy layout on the wire, whatever layout the data uses. + assertEquals(ZxidUtils.makeZxid(1, 0), qp.getZxid()); + // The leader switched at its new epoch and announces the + // switch epoch after the protocol version. + assertEquals(12, qp.getData().length); + ByteBuffer leaderInfoData = ByteBuffer.wrap(qp.getData()); + assertEquals(LearnerHandler.WIDE_COUNTER_PROTOCOL_VERSION, leaderInfoData.getInt()); + assertEquals(1, leaderInfoData.getLong()); + assertTrue(l.self.getZxidLayoutState().isSwitched()); + + qp = new QuorumPacket(Leader.ACKEPOCH, 0, new byte[4], null); + oa.writeRecord(qp, null); + + readPacketSkippingPing(ia, qp); + assertEquals(Leader.DIFF, qp.getType()); + + readPacketSkippingPing(ia, qp); + assertEquals(Leader.NEWLEADER, qp.getType()); + // The NEWLEADER zxid is a real zxid, composed with the + // wide-counter layout from the switch epoch on. + assertEquals(ZxidLayout.WIDE_COUNTER.makeZxid(1, 0), qp.getZxid()); + assertCurrentEpochGotUpdated(1, l.self, ClientBase.CONNECTION_TIMEOUT); + + qp = new QuorumPacket(Leader.ACK, qp.getZxid(), null, null); + oa.writeRecord(qp, null); + + readPacketSkippingPing(ia, qp); + assertEquals(Leader.UPTODATE, qp.getType()); + } + }, testData); + } finally { + System.clearProperty(QuorumPeer.WIDE_COUNTER_ZXID_ENABLED); + } + } + + @Test + public void testRefuseOldProtocolLearnerAfterWideCounterSwitch(@TempDir File testData) throws Exception { + System.setProperty(QuorumPeer.WIDE_COUNTER_ZXID_ENABLED, "true"); + try { + testLeaderConversation(new LeaderConversation() { + public void converseWithLeader(InputArchive ia, OutputArchive oa, Leader l) throws IOException { + // A learner announcing a protocol version below + // WIDE_COUNTER_PROTOCOL_VERSION cannot understand the + // switched layout and must be refused. + LearnerInfo li = new LearnerInfo(1, 0x10000, 0); + byte[] liBytes = RequestRecord.fromRecord(li).readBytes(); + QuorumPacket qp = new QuorumPacket(Leader.FOLLOWERINFO, 0, liBytes, null); + oa.writeRecord(qp, null); + try { + readPacketSkippingPing(ia, qp); + fail("expected the leader to drop an old-protocol learner after the layout switch"); + } catch (IOException expected) { + // connection closed by the leader + } + } + }, testData); + } finally { + System.clearProperty(QuorumPeer.WIDE_COUNTER_ZXID_ENABLED); + } + } + + @Test + public void testRefusePre0x10000LearnerAfterWideCounterSwitch(@TempDir File testData) throws Exception { + System.setProperty(QuorumPeer.WIDE_COUNTER_ZXID_ENABLED, "true"); + try { + testLeaderConversation(new LeaderConversation() { + public void converseWithLeader(InputArchive ia, OutputArchive oa, Leader l) throws IOException { + // A pre-3.4.6 learner sends an 8-byte sid-only FOLLOWERINFO + // payload (no protocol-version int), so its version stays at + // the default 0x1, i.e. below 0x10000. Once the ensemble has + // switched to the wide-counter layout it still cannot decode + // the wide zxids and must be refused, not advanced through + // the legacy extrapolation branch. + byte[] sidOnly = new byte[8]; + ByteBuffer.wrap(sidOnly).putLong(1L); + QuorumPacket qp = new QuorumPacket(Leader.FOLLOWERINFO, 0, sidOnly, null); + oa.writeRecord(qp, null); + try { + readPacketSkippingPing(ia, qp); + fail("expected the leader to drop a pre-0x10000 learner after the layout switch"); + } catch (IOException expected) { + // connection closed by the leader + } + } + }, testData); + } finally { + System.clearProperty(QuorumPeer.WIDE_COUNTER_ZXID_ENABLED); + } + } + @Test public void testTxnTimeout(@TempDir File testData) throws Exception { testLeaderConversation(new LeaderConversation() { @@ -999,7 +1156,7 @@ public void converseWithLeader(InputArchive ia, OutputArchive oa, Leader l) thro readPacketSkippingPing(ia, qp); assertEquals(Leader.LEADERINFO, qp.getType()); assertEquals(ZxidUtils.makeZxid(1, 0), qp.getZxid()); - assertEquals(ByteBuffer.wrap(qp.getData()).getInt(), 0x10000); + assertEquals(ByteBuffer.wrap(qp.getData()).getInt(), LearnerHandler.WIDE_COUNTER_PROTOCOL_VERSION); assertEquals(1, l.self.getAcceptedEpoch()); assertEquals(0, l.self.getCurrentEpoch()); @@ -1081,7 +1238,7 @@ public void converseWithObserver(InputArchive ia, OutputArchive oa, Observer o) assertEquals(qp.getZxid(), 0); LearnerInfo learnInfo = new LearnerInfo(); ByteBufferInputStream.byteBuffer2Record(ByteBuffer.wrap(qp.getData()), learnInfo); - assertEquals(learnInfo.getProtocolVersion(), 0x10000); + assertEquals(learnInfo.getProtocolVersion(), LearnerHandler.WIDE_COUNTER_PROTOCOL_VERSION); assertEquals(learnInfo.getServerid(), 0); // We are simulating an established leader, so the epoch is 1 @@ -1201,7 +1358,7 @@ public void converseWithLeader(InputArchive ia, OutputArchive oa, Leader l) thro readPacketSkippingPing(ia, qp); assertEquals(Leader.LEADERINFO, qp.getType()); assertEquals(ZxidUtils.makeZxid(21, 0), qp.getZxid()); - assertEquals(ByteBuffer.wrap(qp.getData()).getInt(), 0x10000); + assertEquals(ByteBuffer.wrap(qp.getData()).getInt(), LearnerHandler.WIDE_COUNTER_PROTOCOL_VERSION); qp = new QuorumPacket(Leader.ACKEPOCH, 0, new byte[4], null); oa.writeRecord(qp, null); readPacketSkippingPing(ia, qp); @@ -1237,7 +1394,7 @@ public void converseWithLeader(InputArchive ia, OutputArchive oa, Leader l) thro readPacketSkippingPing(ia, qp); assertEquals(Leader.LEADERINFO, qp.getType()); assertEquals(ZxidUtils.makeZxid(1, 0), qp.getZxid()); - assertEquals(ByteBuffer.wrap(qp.getData()).getInt(), 0x10000); + assertEquals(ByteBuffer.wrap(qp.getData()).getInt(), LearnerHandler.WIDE_COUNTER_PROTOCOL_VERSION); Thread.sleep(l.self.getInitLimit() * l.self.getTickTime() + 5000); // The leader didn't get a quorum of acks - make sure that leader's current epoch is not advanced @@ -1300,7 +1457,9 @@ private ConversableObserver createObserver(File tmpDir, QuorumPeer peer) throws } private String readContentsOfFile(File f) throws IOException { - return new BufferedReader(new FileReader(f)).readLine(); + try (BufferedReader reader = new BufferedReader(new FileReader(f))) { + return reader.readLine(); + } } @Test @@ -1308,8 +1467,9 @@ public void testInitialAcceptedCurrent(@TempDir File testData) throws Exception File tmpDir = File.createTempFile("test", ".dir", testData); tmpDir.delete(); tmpDir.mkdir(); + FileTxnSnapLog logFactory = null; try { - FileTxnSnapLog logFactory = new FileTxnSnapLog(tmpDir, tmpDir); + logFactory = new FileTxnSnapLog(tmpDir, tmpDir); File version2 = new File(tmpDir, "version-2"); version2.mkdir(); logFactory.save(new DataTree(), new ConcurrentHashMap<>(), false); @@ -1326,6 +1486,11 @@ public void testInitialAcceptedCurrent(@TempDir File testData) throws Exception assertEquals(3, Integer.parseInt(readContentsOfFile(new File(version2, QuorumPeer.CURRENT_EPOCH_FILENAME)))); assertEquals(3, Integer.parseInt(readContentsOfFile(new File(version2, QuorumPeer.ACCEPTED_EPOCH_FILENAME)))); } finally { + if (logFactory != null) { + // release the txn log file handles, otherwise the temp + // directory cannot be deleted on Windows + logFactory.close(); + } TestUtils.deleteFileRecursively(tmpDir); } } diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/util/LogChopperTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/util/LogChopperTest.java new file mode 100644 index 00000000000..c59ecaf6339 --- /dev/null +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/util/LogChopperTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.util; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.Adler32; +import java.util.zip.Checksum; +import org.apache.jute.BinaryOutputArchive; +import org.apache.zookeeper.ZKTestCase; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.server.persistence.FileHeader; +import org.apache.zookeeper.server.persistence.FileTxnLog; +import org.apache.zookeeper.server.persistence.Util; +import org.apache.zookeeper.txn.ErrorTxn; +import org.apache.zookeeper.txn.TxnHeader; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link LogChopper}, including chopping a log whose zxids span + * the switch to the wide-counter layout (ZOOKEEPER-2789). + * + *

Introduced by Benedict Jin (asdf2014) for ZOOKEEPER-2789. + */ +public class LogChopperTest extends ZKTestCase { + + private byte[] buildTxnLog(long... zxids) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BinaryOutputArchive oa = BinaryOutputArchive.getArchive(baos); + new FileHeader(FileTxnLog.TXNLOG_MAGIC, 2, 1).serialize(oa, "fileheader"); + for (long zxid : zxids) { + TxnHeader hdr = new TxnHeader(1, 1, zxid, 1, ZooDefs.OpCode.error); + byte[] bytes = Util.marshallTxnEntry(hdr, new ErrorTxn(1), null); + Checksum crc = new Adler32(); + crc.update(bytes, 0, bytes.length); + oa.writeLong(crc.getValue(), "crcvalue"); + oa.writeBuffer(bytes, "txnEntry"); + oa.writeByte((byte) 'B', "EOR"); + } + return baos.toByteArray(); + } + + @Test + public void testChopLegacyLog() throws IOException { + // An intra-epoch gap between (1, 2) and (1, 4) exercises the gap + // diagnostics with the default legacy-only layout. + byte[] log = buildTxnLog( + ZxidUtils.makeZxid(1, 1), + ZxidUtils.makeZxid(1, 2), + ZxidUtils.makeZxid(1, 4), + ZxidUtils.makeZxid(2, 1)); + ByteArrayOutputStream chopped = new ByteArrayOutputStream(); + assertTrue(LogChopper.chop(new ByteArrayInputStream(log), chopped, ZxidUtils.makeZxid(1, 4))); + } + + @Test + public void testChopAcrossZxidLayoutSwitch() throws IOException { + // The log spans the switch to the wide-counter layout at epoch 6: + // legacy zxids of epoch 5, then wide-counter zxids of epoch 6, with + // gaps on both sides of the boundary so the diagnostics decompose + // zxids of both layouts. + ZxidLayoutState layoutState = new ZxidLayoutState(); + layoutState.switchAt(6); + byte[] log = buildTxnLog( + ZxidUtils.makeZxid(5, 1), + ZxidUtils.makeZxid(5, 3), + ZxidLayout.WIDE_COUNTER.makeZxid(6, 1), + ZxidLayout.WIDE_COUNTER.makeZxid(6, 3), + ZxidLayout.WIDE_COUNTER.makeZxid(7, 1)); + ByteArrayOutputStream chopped = new ByteArrayOutputStream(); + assertTrue(LogChopper.chop( + new ByteArrayInputStream(log), chopped, ZxidLayout.WIDE_COUNTER.makeZxid(6, 3), layoutState)); + } + + @Test + public void testChopReturnsFalseWhenZxidMissing() throws IOException { + byte[] log = buildTxnLog( + ZxidUtils.makeZxid(1, 1), + ZxidUtils.makeZxid(1, 2)); + ByteArrayOutputStream chopped = new ByteArrayOutputStream(); + // The log ends before any zxid larger than the requested one shows + // up, so nothing is chopped. + assertFalse(LogChopper.chop(new ByteArrayInputStream(log), chopped, ZxidUtils.makeZxid(1, 5))); + } + +} diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/util/ZxidLayoutStateTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/util/ZxidLayoutStateTest.java new file mode 100644 index 00000000000..728698b2e72 --- /dev/null +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/util/ZxidLayoutStateTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import org.apache.zookeeper.ZKTestCase; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link ZxidLayoutState}: layout selection for new and historic + * zxids around the switch epoch, and the switch/adopt/clear life cycle. + * + *

Introduced by Benedict Jin (asdf2014) for ZOOKEEPER-2789. + */ +public class ZxidLayoutStateTest extends ZKTestCase { + + @Test + public void testNotSwitchedDefaults() { + ZxidLayoutState state = new ZxidLayoutState(); + assertFalse(state.isSwitched()); + assertEquals(ZxidLayoutState.NOT_SWITCHED, state.getSwitchEpoch()); + assertSame(ZxidLayout.LEGACY, state.current()); + // Without a switch everything is legacy, however large the zxid. + assertSame(ZxidLayout.LEGACY, state.layoutFor(0L)); + assertSame(ZxidLayout.LEGACY, state.layoutFor(ZxidUtils.makeZxid(5, 1000))); + assertSame(ZxidLayout.LEGACY, state.layoutFor(Long.MAX_VALUE)); + } + + @Test + public void testSwitchAt() { + ZxidLayoutState state = new ZxidLayoutState(); + state.switchAt(6); + assertTrue(state.isSwitched()); + assertEquals(6, state.getSwitchEpoch()); + assertSame(ZxidLayout.WIDE_COUNTER, state.current()); + // Re-recording the same switch epoch is fine. + state.switchAt(6); + assertEquals(6, state.getSwitchEpoch()); + } + + @Test + public void testLayoutForAroundSwitchPoint() { + ZxidLayoutState state = new ZxidLayoutState(); + state.switchAt(6); + long firstWide = ZxidLayout.WIDE_COUNTER.makeZxid(6, 0); + // Everything below the first wide-counter zxid is legacy data. + assertSame(ZxidLayout.LEGACY, state.layoutFor(0L)); + assertSame(ZxidLayout.LEGACY, state.layoutFor(ZxidUtils.makeZxid(5, 1000))); + assertSame(ZxidLayout.LEGACY, state.layoutFor(ZxidUtils.makeZxid(5, 0xffffffffL))); + assertSame(ZxidLayout.LEGACY, state.layoutFor(firstWide - 1)); + // The switch point itself and everything above is wide-counter. + assertSame(ZxidLayout.WIDE_COUNTER, state.layoutFor(firstWide)); + assertSame(ZxidLayout.WIDE_COUNTER, state.layoutFor(ZxidLayout.WIDE_COUNTER.makeZxid(6, 1))); + assertSame(ZxidLayout.WIDE_COUNTER, state.layoutFor(ZxidLayout.WIDE_COUNTER.makeZxid(7, 42))); + } + + @Test + public void testLegacyNewEpochZxidNotMistakenForWide() { + // A legacy new-epoch zxid (counter == 0 in the legacy layout) below + // the switch point must be decomposed with the legacy layout: with + // the wide layout its counter part would read as non-zero and the + // DIFF/TRUNC decisions in LearnerHandler.syncFollower() would break. + ZxidLayoutState state = new ZxidLayoutState(); + state.switchAt(6); + long legacyNewEpoch = ZxidUtils.makeZxid(5, 0); + assertEquals(0, state.layoutFor(legacyNewEpoch).getCounterFromZxid(legacyNewEpoch)); + assertEquals(5, state.layoutFor(legacyNewEpoch).getEpochFromZxid(legacyNewEpoch)); + } + + @Test + public void testSwitchAtRejectsConflict() { + ZxidLayoutState state = new ZxidLayoutState(); + state.switchAt(6); + assertThrows(IllegalStateException.class, () -> state.switchAt(7)); + assertEquals(6, state.getSwitchEpoch()); + } + + @Test + public void testSwitchAtRejectsOutOfRange() { + ZxidLayoutState state = new ZxidLayoutState(); + assertThrows(IllegalArgumentException.class, () -> state.switchAt(-1)); + assertThrows(IllegalArgumentException.class, () -> state.switchAt(ZxidLayout.WIDE_COUNTER.getMaxEpoch() + 1)); + assertFalse(state.isSwitched()); + } + + @Test + public void testForceSwitchAt() { + ZxidLayoutState state = new ZxidLayoutState(); + state.switchAt(6); + // An unused switch may be moved when the next leader announces a + // different switch epoch (the previous announcing leader failed + // before its epoch committed anything). + state.forceSwitchAt(7); + assertEquals(7, state.getSwitchEpoch()); + } + + @Test + public void testClear() { + ZxidLayoutState state = new ZxidLayoutState(); + state.switchAt(6); + state.clear(); + assertFalse(state.isSwitched()); + assertSame(ZxidLayout.LEGACY, state.current()); + assertSame(ZxidLayout.LEGACY, state.layoutFor(Long.MAX_VALUE)); + } + + @Test + public void testLegacyOnlyIsImmutable() { + ZxidLayoutState state = ZxidLayoutState.legacyOnly(); + assertFalse(state.isSwitched()); + assertSame(ZxidLayout.LEGACY, state.current()); + assertThrows(UnsupportedOperationException.class, () -> state.switchAt(1)); + assertThrows(UnsupportedOperationException.class, () -> state.forceSwitchAt(1)); + assertThrows(UnsupportedOperationException.class, state::clear); + } + +} diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/util/ZxidLayoutTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/util/ZxidLayoutTest.java new file mode 100644 index 00000000000..4d8ec7a643b --- /dev/null +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/util/ZxidLayoutTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import org.apache.zookeeper.ZKTestCase; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link ZxidLayout}: the legacy 32/32 layout and the + * wide-counter 24/40 layout introduced by ZOOKEEPER-2789. + * + *

Introduced by Benedict Jin (asdf2014) for ZOOKEEPER-2789. + */ +public class ZxidLayoutTest extends ZKTestCase { + + @Test + public void testLegacyLayoutConstants() { + assertEquals(32, ZxidLayout.LEGACY.getCounterBits()); + assertEquals(0xffffffffL, ZxidLayout.LEGACY.getMaxCounter()); + assertEquals((1L << 31) - 1, ZxidLayout.LEGACY.getMaxEpoch()); + } + + @Test + public void testWideCounterLayoutConstants() { + assertEquals(40, ZxidLayout.WIDE_COUNTER.getCounterBits()); + assertEquals(0xffffffffffL, ZxidLayout.WIDE_COUNTER.getMaxCounter()); + assertEquals((1L << 23) - 1, ZxidLayout.WIDE_COUNTER.getMaxEpoch()); + } + + @Test + public void testLegacyMatchesZxidUtils() { + long[] zxids = {0L, 1L, ZxidUtils.makeZxid(5, 1000), ZxidUtils.makeZxid(42, 0xffffffffL), -1L}; + for (long zxid : zxids) { + assertEquals(ZxidUtils.getEpochFromZxid(zxid), ZxidLayout.LEGACY.getEpochFromZxid(zxid)); + assertEquals(ZxidUtils.getCounterFromZxid(zxid), ZxidLayout.LEGACY.getCounterFromZxid(zxid)); + } + assertEquals(ZxidUtils.makeZxid(5, 1000), ZxidLayout.LEGACY.makeZxid(5, 1000)); + } + + @Test + public void testEpochCounterRoundTrip() { + for (ZxidLayout layout : new ZxidLayout[]{ZxidLayout.LEGACY, ZxidLayout.WIDE_COUNTER}) { + long[] epochs = {0L, 1L, 5L, 0xabcdL, layout.getMaxEpoch()}; + long[] counters = {0L, 1L, 1000L, layout.getMaxCounter()}; + for (long epoch : epochs) { + for (long counter : counters) { + long zxid = layout.makeZxid(epoch, counter); + assertTrue(zxid >= 0, "zxid must stay non-negative for epoch <= getMaxEpoch()"); + assertEquals(epoch, layout.getEpochFromZxid(zxid), layout + " epoch of 0x" + Long.toHexString(zxid)); + assertEquals(counter, layout.getCounterFromZxid(zxid), layout + " counter of 0x" + Long.toHexString(zxid)); + } + } + } + } + + @Test + public void testMakeZxidMasksCounterOverflow() { + for (ZxidLayout layout : new ZxidLayout[]{ZxidLayout.LEGACY, ZxidLayout.WIDE_COUNTER}) { + // A counter wider than its bits must not corrupt the epoch part. + assertEquals(layout.makeZxid(1, 0), layout.makeZxid(1, layout.getMaxCounter() + 1)); + assertEquals(layout.makeZxid(1, 1), layout.makeZxid(1, layout.getMaxCounter() + 2)); + } + } + + @Test + public void testClearCounter() { + for (ZxidLayout layout : new ZxidLayout[]{ZxidLayout.LEGACY, ZxidLayout.WIDE_COUNTER}) { + assertEquals(0L, layout.clearCounter(0L)); + assertEquals(layout.makeZxid(5, 0), layout.clearCounter(layout.makeZxid(5, 12345))); + // Clearing the counter and OR-ing a new one in composes a valid + // zxid, which is how zookeeper.testingonly.initialZxid is applied. + long zxid = layout.makeZxid(3, 777); + assertEquals(layout.makeZxid(3, 42), layout.clearCounter(zxid) | 42L); + } + assertEquals(0xffffffff00000000L, ZxidLayout.LEGACY.clearCounter(-1L)); + assertEquals(0xffffff0000000000L, ZxidLayout.WIDE_COUNTER.clearCounter(-1L)); + } + + @Test + public void testCounterRolloverBoundary() { + // Leader.propose() forces a re-election when the counter is + // exhausted: the check compares the counter part to getMaxCounter(). + for (ZxidLayout layout : new ZxidLayout[]{ZxidLayout.LEGACY, ZxidLayout.WIDE_COUNTER}) { + long nearRollover = layout.makeZxid(1, layout.getMaxCounter() - 1); + long atRollover = layout.makeZxid(1, layout.getMaxCounter()); + assertNotEquals(layout.getMaxCounter(), layout.getCounterFromZxid(nearRollover)); + assertEquals(layout.getMaxCounter(), layout.getCounterFromZxid(atRollover)); + // The exhausted counter does not leak into the epoch part. + assertEquals(1L, layout.getEpochFromZxid(atRollover)); + } + } + + @Test + public void testOrderPreservedAcrossLayoutSwitch() { + // The switch always comes with an epoch bump, so the first + // wide-counter zxid is numerically larger than every legacy zxid of + // the previous epochs — the total order of zxids survives the switch. + long lastLegacy = ZxidLayout.LEGACY.makeZxid(5, ZxidLayout.LEGACY.getMaxCounter()); + long firstWide = ZxidLayout.WIDE_COUNTER.makeZxid(6, 0); + assertTrue(firstWide > lastLegacy); + } + + @Test + public void testToString() { + assertEquals("legacy", ZxidLayout.LEGACY.toString()); + assertEquals("wide-counter", ZxidLayout.WIDE_COUNTER.toString()); + } + +} diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/util/ZxidUtilsTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/util/ZxidUtilsTest.java new file mode 100644 index 00000000000..7fccdb61ceb --- /dev/null +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/util/ZxidUtilsTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import org.apache.zookeeper.ZKTestCase; +import org.junit.jupiter.api.Test; + +/** + * Tests for the static {@link ZxidUtils} helpers, which are fixed to the + * legacy 32-bit epoch / 32-bit counter layout (see {@link ZxidLayout}). + * + *

Introduced by Benedict Jin (asdf2014) for ZOOKEEPER-2789. + */ +public class ZxidUtilsTest extends ZKTestCase { + + @Test + public void testMakeZxid() { + assertEquals(0L, ZxidUtils.makeZxid(0, 0)); + assertEquals(1L, ZxidUtils.makeZxid(0, 1)); + assertEquals(0x100000000L, ZxidUtils.makeZxid(1, 0)); + assertEquals(0x100000001L, ZxidUtils.makeZxid(1, 1)); + assertEquals(0x5000003e8L, ZxidUtils.makeZxid(5, 1000)); + // A counter wider than its 32 bits must not corrupt the epoch part. + assertEquals(ZxidUtils.makeZxid(1, 0), ZxidUtils.makeZxid(1, 0x100000000L)); + } + + @Test + public void testGetEpochFromZxid() { + assertEquals(0L, ZxidUtils.getEpochFromZxid(0L)); + assertEquals(0L, ZxidUtils.getEpochFromZxid(0xffffffffL)); + assertEquals(1L, ZxidUtils.getEpochFromZxid(ZxidUtils.makeZxid(1, 0))); + assertEquals(5L, ZxidUtils.getEpochFromZxid(ZxidUtils.makeZxid(5, 1000))); + } + + @Test + public void testGetCounterFromZxid() { + assertEquals(0L, ZxidUtils.getCounterFromZxid(0L)); + assertEquals(0L, ZxidUtils.getCounterFromZxid(ZxidUtils.makeZxid(7, 0))); + assertEquals(1000L, ZxidUtils.getCounterFromZxid(ZxidUtils.makeZxid(5, 1000))); + assertEquals(0xffffffffL, ZxidUtils.getCounterFromZxid(-1L)); + } + + @Test + public void testEpochCounterRoundTrip() { + long[] epochs = {0L, 1L, 5L, 0xabcdL}; + long[] counters = {0L, 1L, 1000L, 0xffffffffL}; + for (long epoch : epochs) { + for (long counter : counters) { + long zxid = ZxidUtils.makeZxid(epoch, counter); + assertEquals(epoch, ZxidUtils.getEpochFromZxid(zxid), "epoch of zxid " + Long.toHexString(zxid)); + assertEquals(counter, ZxidUtils.getCounterFromZxid(zxid), "counter of zxid " + Long.toHexString(zxid)); + } + } + } + + @Test + public void testZxidToString() { + assertEquals("0", ZxidUtils.zxidToString(0L)); + assertEquals("ff", ZxidUtils.zxidToString(255L)); + assertEquals("100000000", ZxidUtils.zxidToString(ZxidUtils.makeZxid(1, 0))); + } + +} diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/FollowerResyncConcurrencyTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/FollowerResyncConcurrencyTest.java index 40cd03bd972..abe8a273fbe 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/FollowerResyncConcurrencyTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/FollowerResyncConcurrencyTest.java @@ -43,6 +43,7 @@ import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.server.ZKDatabase; import org.apache.zookeeper.server.quorum.Leader; +import org.apache.zookeeper.server.util.ZxidUtils; import org.apache.zookeeper.test.ClientBase.CountdownWatcher; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -562,8 +563,8 @@ private static TestableZooKeeper createTestableClient( private void verifyState(QuorumUtil qu, int index, Leader leader) { LOG.info("Verifying state"); assertTrue(qu.getPeer(index).peer.follower != null, "Not following"); - long epochF = (qu.getPeer(index).peer.getActiveServer().getZxid() >> 32L); - long epochL = (leader.getEpoch() >> 32L); + long epochF = ZxidUtils.getEpochFromZxid(qu.getPeer(index).peer.getActiveServer().getZxid()); + long epochL = ZxidUtils.getEpochFromZxid(leader.getEpoch()); assertTrue(epochF == epochL, "Zxid: " + qu.getPeer(index).peer.getActiveServer().getZKDatabase().getDataTreeLastProcessedZxid() + "Current epoch: " + epochF); diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReconfigTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReconfigTest.java index 1615be17242..e8e5bdaec80 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReconfigTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReconfigTest.java @@ -58,6 +58,7 @@ import org.apache.zookeeper.server.quorum.flexible.QuorumHierarchical; import org.apache.zookeeper.server.quorum.flexible.QuorumMaj; import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; +import org.apache.zookeeper.server.util.ZxidUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -934,7 +935,7 @@ public void testInitialConfigHasPositiveVersion() throws Exception { String configStr = testServerHasConfig(zkArr[i], null, null); QuorumVerifier qv = qu.getPeer(i).peer.configFromString(configStr); long version = qv.getVersion(); - assertTrue(version == 0x100000000L); + assertTrue(version == ZxidUtils.makeZxid(1, 0)); } }