diff --git a/VotingPlugin/pom.xml b/VotingPlugin/pom.xml
index 5a74489bb..4204d571d 100644
--- a/VotingPlugin/pom.xml
+++ b/VotingPlugin/pom.xml
@@ -156,6 +156,10 @@
${project.groupId}.votingplugin.bstats
+
+ org.bouncycastle
+ ${project.groupId}.votingplugin.bouncycastle
+
xyz.upperlevel.spigot
@@ -209,12 +213,17 @@
com.bencodez:simpleapi
org/bouncycastle/**
- com/bencodez/simpleapi/servercomm/http/**
redis/clients/**
org/eclipse/paho/**
META-INF/versions/**
+
+ org.bouncycastle:*
+
+ META-INF/versions/25/**
+
+
*:*
@@ -433,6 +442,12 @@
2.14.0
provided
+
+ com.bencodez
+ simpleapi
+ 1.0.2-SNAPSHOT
+ compile
+
org.junit.jupiter
junit-jupiter-engine
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java
index 4f34d41c9..824bf0af0 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java
@@ -152,6 +152,11 @@ public class VotingPluginMain extends AdvancedCorePlugin {
@Getter
public static VotingPluginMain plugin;
+ /** Exact backend plugin file name Bukkit uses when consuming its update folder. */
+ public File getLoadedPluginJarFile() {
+ return getFile();
+ }
+
@Getter
@Setter
private ArrayList adminVoteCommand;
@@ -167,6 +172,7 @@ public class VotingPluginMain extends AdvancedCorePlugin {
private final ProcessedVoteCache backendProcessedVoteCache = new ProcessedVoteCache();
private final AtomicReference backendPluginMessageTarget = new AtomicReference<>();
private PluginMessageHandler backendPluginMessageRelay;
+ private com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessage backendPluginMessageRelayOwner;
private volatile BackendControlAutoEnrollment backendControlAutoEnrollment;
private volatile BackendControlConnector backendControlConnector;
private final ScheduledExecutorService backendControlConnectorLifecycle = Executors.newSingleThreadScheduledExecutor(runnable -> {
@@ -1224,29 +1230,261 @@ public String getBackendHostedControlStatus() {
}
/** Recreates proxy transports after Control applies BungeeSettings.yml. */
- public synchronized void restartBackendProxyHandler() {
+ public void restartBackendProxyHandler() {
+ restartBackendProxyHandler(System.nanoTime() + TimeUnit.SECONDS.toNanos(25));
+ }
+
+ /** Recreates proxy transports while preserving the caller's end-to-end validation deadline. */
+ public void restartBackendProxyHandler(long validationDeadlineNanos) {
+ BackendProxyRestart restart = prepareBackendProxyHandlerRestart();
+ try {
+ validateBackendProxyHandlerRestart(restart, validationDeadlineNanos);
+ completeBackendProxyHandlerRestart(restart);
+ } catch (RuntimeException failure) {
+ abortBackendProxyHandlerRestart(restart);
+ throw failure;
+ }
+ }
+
+ /** Constructed on the Bukkit thread, prepared and validated off-thread, then atomically published on Bukkit. */
+ public static final class BackendProxyRestart {
+ private final BackendProxyHandler previous;
+ private final BackendProxyHandler replacement;
+ private BungeeMethod replacementMethod;
+ private final boolean disabled;
+ private final boolean previousRequiresPreparation;
+ private volatile boolean previousPrepared;
+ private boolean presenceStoppedForDisablePreparation;
+ // Redis listener retirement can wait for callbacks and listener shutdown. It is
+ // completed by the Control worker during validation, before the final Bukkit
+ // publication callback, and must be restored if that publication is abandoned.
+ private volatile boolean redisHandoffCompleted;
+ private boolean redisHandoffInProgress;
+ private boolean finished;
+ private boolean abandonmentRequested;
+ // Same-method socket and MQTT replacements must not start a second runtime
+ // endpoint before worker-side preparation retires the predecessor.
+ private boolean replacementLoadDeferred;
+ private volatile boolean published;
+
+ private BackendProxyRestart(BackendProxyHandler previous, BackendProxyHandler replacement, boolean disabled,
+ boolean previousRequiresPreparation) {
+ this.previous = previous;
+ this.replacement = replacement;
+ this.disabled = disabled;
+ this.previousRequiresPreparation = previousRequiresPreparation;
+ }
+
+ /** Only exclusive same-method transports may be restored by the Control worker. */
+ public boolean requiresWorkerRollback() {
+ return previousPrepared && previous != null && replacement != null
+ && ((previous.getMethod() == BungeeMethod.SOCKETS && effectiveReplacementMethod() == BungeeMethod.SOCKETS)
+ || (previous.getMethod() == BungeeMethod.MQTT
+ && effectiveReplacementMethod() == BungeeMethod.MQTT));
+ }
+
+ private BungeeMethod effectiveReplacementMethod() {
+ BungeeMethod loadedMethod = replacement.getMethod();
+ return loadedMethod != null ? loadedMethod : replacementMethod;
+ }
+ }
+
+ public static final class BackendProxyRestartPreparationException extends RuntimeException {
+ private static final long serialVersionUID = 1L;
+ private final BackendProxyRestart restart;
+
+ private BackendProxyRestartPreparationException(BackendProxyRestart restart, RuntimeException cause) {
+ super(cause);
+ this.restart = restart;
+ }
+
+ public BackendProxyRestart restart() { return restart; }
+ }
+
+ public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() {
BackendProxyHandler previous = backendProxyHandler;
if (!bungeeSettings.isUseBungeecoord()) {
- backendProxyHandler = null;
- if (previous != null) previous.close();
- BackendControlAutoEnrollment enrollment = backendControlAutoEnrollment;
- backendControlAutoEnrollment = null;
- if (enrollment != null) enrollment.close();
- return;
+ boolean previousRequiresPreparation = previous != null
+ && (previous.requiresPreparationForReplacement() || previous.requiresRedisRetirement());
+ return new BackendProxyRestart(previous, null, true, previousRequiresPreparation);
}
BungeeMethod replacementMethod = BungeeMethod.getByName(bungeeSettings.getBungeeMethod());
- if (previous != null) previous.prepareForReplacement(replacementMethod);
+ boolean sameSocketReplacement = previous != null && previous.getMethod() == BungeeMethod.SOCKETS
+ && replacementMethod == BungeeMethod.SOCKETS;
+ boolean sameMqttReplacement = previous != null && previous.getMethod() == BungeeMethod.MQTT
+ && replacementMethod == BungeeMethod.MQTT;
+ boolean deferredReplacementLoad = sameSocketReplacement || sameMqttReplacement;
+ // Every transition away from an active HTTP transport must first drain its
+ // durable outgoing queue. Restricting preparation to HTTP-to-HTTP swaps can
+ // strand accepted deliveries when another transport is published.
+ boolean previousRequiresPreparation = previous != null
+ && (deferredReplacementLoad || previous.requiresPreparationForReplacement()
+ || previous.requiresRedisRetirement());
BackendProxyHandler replacement = new BackendProxyHandler(this, backendProcessedVoteCache);
- try {
- replacement.load();
- replacement.validateTransport();
- if (previous != null) previous.completeRedisHandoff(replacement);
- } catch (RuntimeException failure) {
- replacement.close();
- throw failure;
+ if (!deferredReplacementLoad) {
+ try {
+ replacement.loadForReplacement();
+ } catch (RuntimeException failure) {
+ replacement.close();
+ throw failure;
+ }
+ }
+ BackendProxyRestart restart = new BackendProxyRestart(previous, replacement, false, previousRequiresPreparation);
+ restart.replacementMethod = replacementMethod;
+ restart.replacementLoadDeferred = deferredReplacementLoad;
+ return restart;
+ }
+
+ public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long validationDeadlineNanos) {
+ if (restart == null) throw new IllegalArgumentException("Backend proxy restart is required");
+ // Disabling always stops a live predecessor's presence transport. That stop
+ // can perform network/backend work for MQTT, MySQL, and socket transports
+ // even when they have no HTTP/plugin-message handoff to prepare. Keep it on
+ // the Control worker rather than falling through to Bukkit publication.
+ if (restart.disabled && restart.previous != null && !restart.presenceStoppedForDisablePreparation) {
+ restart.presenceStoppedForDisablePreparation = true;
+ restart.previous.preparePresenceForDisable(validationDeadlineNanos);
+ }
+ if (restart.previousRequiresPreparation && !restart.previousPrepared) {
+ // Preparation can close a retrying enrollment transport before a bounded
+ // worker join fails. Mark the restart first so abort/await still owns the
+ // restoration path after a partially completed preparation.
+ restart.previousPrepared = true;
+ BungeeMethod replacementMethod = restart.replacement == null ? null : restart.effectiveReplacementMethod();
+ if (restart.replacement != null) restart.replacement.beginPreparedHttpHandoff();
+ if (!restart.previous.prepareForReplacement(replacementMethod, validationDeadlineNanos))
+ throw new IllegalStateException("Previous proxy transport could not be prepared for replacement");
+ if (replacementMethod == BungeeMethod.HTTP)
+ restart.previous.reservePreparedHttpHandoff(restart.replacement);
+ }
+ if (restart.replacement != null && restart.replacementLoadDeferred) {
+ restart.replacement.loadForReplacement();
+ restart.replacementLoadDeferred = false;
+ }
+ if (restart.replacement != null) restart.replacement.validateTransport(validationDeadlineNanos);
+ // Same-Redis handoff fences callbacks and joins the retiring listener. Those
+ // waits are bounded, but they must never run on the Bukkit publication task.
+ // The staged replacement still keeps all inbound callbacks behind its
+ // publication gate, so doing this on the Control worker cannot expose it early.
+ if (restart.previous != null && restart.replacement != null
+ && restart.previous.requiresRedisHandoff(restart.replacement)) {
+ synchronized (restart) {
+ if (restart.redisHandoffCompleted) return;
+ if (restart.redisHandoffInProgress)
+ throw new IllegalStateException("Redis handoff validation is already in progress");
+ restart.redisHandoffInProgress = true;
+ }
+ boolean abandonAfterHandoff;
+ try {
+ restart.previous.completeRedisHandoff(restart.replacement);
+ } catch (RuntimeException handoffFailure) {
+ synchronized (restart) {
+ restart.redisHandoffInProgress = false;
+ restart.notifyAll();
+ }
+ throw handoffFailure;
+ }
+ synchronized (restart) {
+ restart.redisHandoffInProgress = false;
+ restart.redisHandoffCompleted = true;
+ abandonAfterHandoff = restart.abandonmentRequested;
+ restart.notifyAll();
+ }
+ if (abandonAfterHandoff) abortBackendProxyHandlerRestart(restart);
+ }
+ }
+
+ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) {
+ synchronized (this) {
+ if (restart == null || restart.finished) throw new IllegalStateException("Backend proxy restart is no longer active");
+ boolean requiresRedisHandoff = restart.previous != null && restart.replacement != null
+ && restart.previous.requiresRedisHandoff(restart.replacement);
+ boolean abandonmentRequested;
+ synchronized (restart) {
+ if (requiresRedisHandoff && (!restart.redisHandoffCompleted || restart.redisHandoffInProgress)) {
+ throw new IllegalStateException("Redis handoff must complete during validation before publication");
+ }
+ abandonmentRequested = restart.abandonmentRequested;
+ }
+ if (abandonmentRequested) {
+ abortBackendProxyHandlerRestart(restart);
+ return;
+ }
+ if (backendProxyHandler != restart.previous) throw new IllegalStateException("Backend proxy handler changed during restart");
+ if (restart.disabled) {
+ if (restart.previous != null && !restart.presenceStoppedForDisablePreparation)
+ throw new IllegalStateException("Backend proxy disable must be prepared before Bukkit publication");
+ if (restart.previous != null && !restart.previous.commitPreparedDisable())
+ throw new IllegalStateException("Backend proxy transport accepted a delivery while disabling");
+ backendProxyHandler = null;
+ BackendControlAutoEnrollment enrollment = backendControlAutoEnrollment;
+ backendControlAutoEnrollment = null;
+ restart.finished = true;
+ restart.published = true;
+ closePublishedPreviousBackendProxyHandler(restart.previous, "after disabling");
+ if (enrollment != null) {
+ try {
+ enrollment.close();
+ } catch (RuntimeException cleanupFailure) {
+ getLogger().warning("Backend Control enrollment did not stop cleanly after disabling");
+ debug(cleanupFailure);
+ }
+ }
+ return;
+ }
+ publishBackendProxyHandler(restart.previous, restart.replacement);
+ // Keep the prepared queue owned by the previous handler until every fallible
+ // publication step succeeds. Admission performs no network I/O.
+ try {
+ if (restart.previous != null) restart.previous.completeHttpHandoff(restart.replacement);
+ } catch (RuntimeException handoffFailure) {
+ backendProxyHandler = restart.previous;
+ restart.replacement.abortStagedInboundTo(restart.previous);
+ if (restart.requiresWorkerRollback()) {
+ // The Control worker owns exclusive socket/MQTT teardown and
+ // predecessor reconnection. Publication has only restored the
+ // Bukkit-visible handler and fenced staged inbound callbacks.
+ throw handoffFailure;
+ }
+ // A validated Redis promotion may already own accepted, deduplicated
+ // replay envelopes even though inbound publication has not opened. Move
+ // those envelopes back before closing the staged replacement, whose
+ // normal close path deliberately clears its replay queue.
+ if (restart.redisHandoffCompleted && restart.previous != null) {
+ try {
+ restart.previous.restoreAfterFailedReplacement(restart.replacement);
+ restart.previous.refreshPresenceAfterFailedReplacement();
+ } catch (RuntimeException restorationFailure) {
+ handoffFailure.addSuppressed(restorationFailure);
+ // Retain the staged replacement and its replay queue for the caller's
+ // subsequent rollback retry rather than clearing accepted envelopes.
+ throw handoffFailure;
+ }
+ }
+ if (requiresAsyncStagedReplacementClose(restart))
+ closeStagedRedisReplacementAsync(restart.replacement);
+ else try {
+ restart.replacement.close();
+ } catch (RuntimeException closeFailure) {
+ handoffFailure.addSuppressed(closeFailure);
+ }
+ if (!restart.redisHandoffCompleted) {
+ try {
+ restart.previous.restoreAfterFailedReplacement();
+ restart.previous.refreshPresenceAfterFailedReplacement();
+ } catch (RuntimeException restorationFailure) {
+ handoffFailure.addSuppressed(restorationFailure);
+ }
+ }
+ restart.finished = true;
+ throw handoffFailure;
+ }
+ restart.replacement.activateInboundMessages();
+ restart.replacement.replayRedisAfterHandoffPublication();
+ restart.finished = true;
+ restart.published = true;
+ closePublishedPreviousBackendProxyHandler(restart.previous, "after publication");
}
- backendProxyHandler = replacement;
- if (previous != null) previous.close();
try {
refreshBackendControlAutoEnrollment();
} catch (IOException e) {
@@ -1254,17 +1492,146 @@ public synchronized void restartBackendProxyHandler() {
}
}
+ /** Retires network-backed predecessors without blocking the Bukkit publication callback. */
+ void closePublishedPreviousBackendProxyHandler(BackendProxyHandler previous, String phase) {
+ if (previous == null) return;
+ Thread cleanup = new Thread(() -> closePublishedPreviousBackendProxyHandlerNow(previous, phase),
+ "VotingPlugin-Retired-Backend");
+ cleanup.setDaemon(true);
+ cleanup.start();
+ }
+
+ private void closePublishedPreviousBackendProxyHandlerNow(BackendProxyHandler previous, String phase) {
+ try {
+ previous.close();
+ } catch (RuntimeException cleanupFailure) {
+ // Publication is committed. Cleanup failure must not roll back the only live handler.
+ getLogger().warning("Previous backend proxy handler did not stop cleanly " + phase);
+ debug(cleanupFailure);
+ }
+ }
+
+ /** Publishes the handler before opening any transport callback or presence gate. */
+ void publishBackendProxyHandler(BackendProxyHandler previous, BackendProxyHandler replacement) {
+ backendProxyHandler = replacement;
+ try {
+ replacement.activatePresenceReporting();
+ } catch (RuntimeException activationFailure) {
+ backendProxyHandler = previous;
+ replacement.abortStagedInboundTo(previous);
+ throw activationFailure;
+ }
+ }
+
+ /** Returns false once publication committed and can no longer be rolled back as a failed apply. */
+ public synchronized boolean requestBackendProxyHandlerRestartAbandonment(BackendProxyRestart restart) {
+ if (restart == null) return true;
+ synchronized (restart) {
+ if (restart.published) return false;
+ restart.abandonmentRequested = true;
+ restart.notifyAll();
+ }
+ return true;
+ }
+
+ public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart restart) {
+ if (restart == null || restart.finished) return;
+ synchronized (restart) {
+ // Validation owns the retiring Redis listener until its handoff either
+ // succeeds or fails. Deferring rollback closes the race where abort could
+ // observe a false completion flag and leave that listener fenced.
+ if (restart.redisHandoffInProgress) {
+ restart.abandonmentRequested = true;
+ return;
+ }
+ }
+ RuntimeException cleanupFailure = null;
+ if (restart.redisHandoffCompleted && restart.previous != null && restart.replacement != null) {
+ // Preserve the promoted replacement's pre-publication replay queue before
+ // its close fences and clears it, then retire that staged listener.
+ try {
+ restart.previous.restoreAfterFailedReplacement(restart.replacement);
+ } catch (RuntimeException failure) {
+ cleanupFailure = failure;
+ }
+ }
+ if (restart.replacement != null) {
+ try {
+ restart.replacement.abortStagedInboundTo(restart.previous);
+ if (requiresAsyncStagedReplacementClose(restart)) closeStagedRedisReplacementAsync(restart.replacement);
+ else restart.replacement.close();
+ } catch (RuntimeException failure) {
+ if (cleanupFailure == null) cleanupFailure = failure;
+ else cleanupFailure.addSuppressed(failure);
+ }
+ }
+ if (backendProxyHandler == restart.previous && restart.previous != null
+ && (restart.previousPrepared || restart.redisHandoffCompleted
+ || restart.previous.getMethod() == BungeeMethod.PLUGINMESSAGING)) {
+ if (!restart.redisHandoffCompleted) try {
+ restart.previous.restoreAfterFailedReplacement();
+ } catch (RuntimeException failure) {
+ if (cleanupFailure == null) cleanupFailure = failure;
+ else cleanupFailure.addSuppressed(failure);
+ }
+ }
+ if (backendProxyHandler == restart.previous && restart.previous != null
+ && restart.presenceStoppedForDisablePreparation) {
+ try {
+ restart.previous.restorePresenceAfterFailedDisablePreparation();
+ } catch (RuntimeException failure) {
+ if (cleanupFailure == null) cleanupFailure = failure;
+ else cleanupFailure.addSuppressed(failure);
+ }
+ }
+ restart.finished = true;
+ if (cleanupFailure != null) throw cleanupFailure;
+ }
+
+ private boolean requiresAsyncStagedReplacementClose(BackendProxyRestart restart) {
+ return restart.redisHandoffCompleted || restart.replacement.getMethod() == BungeeMethod.REDIS;
+ }
+
+ /** A staged Redis listener can spend its bounded join timeout in close(); abort runs on Bukkit. */
+ private void closeStagedRedisReplacementAsync(BackendProxyHandler replacement) {
+ Thread cleanup = new Thread(() -> {
+ try {
+ replacement.close();
+ } catch (RuntimeException cleanupFailure) {
+ getLogger().warning("Staged Redis backend proxy handler did not stop cleanly after rollback");
+ debug(cleanupFailure);
+ }
+ }, "VotingPlugin-Staged-Redis-Rollback");
+ cleanup.setDaemon(true);
+ cleanup.start();
+ }
+
+ public void awaitBackendProxyHandlerRollback(BackendProxyRestart restart, long deadlineNanos) {
+ if (restart != null && restart.previousPrepared) {
+ restart.previous.awaitRestoreAfterFailedReplacement(deadlineNanos);
+ }
+ }
+
/** Applies only proxy communication settings without reloading unrelated Bukkit configuration. */
public synchronized void reloadBackendProxyMethodFromControl() {
+ reloadBackendProxyMethodSettingsFromControl();
+ restartBackendProxyHandler();
+ }
+
+ /** Bukkit-side narrow preparation for a proxy-method Control APPLY. */
+ public synchronized BackendProxyRestart prepareBackendProxyMethodRestartFromControl() {
+ reloadBackendProxyMethodSettingsFromControl();
+ return prepareBackendProxyHandlerRestart();
+ }
+
+ private void reloadBackendProxyMethodSettingsFromControl() {
bungeeSettings.reloadData();
getOptions().setServer(bungeeSettings.getServer());
updateAdvancedCoreHook();
- restartBackendProxyHandler();
}
/** Keeps one plugin-message listener for the plugin lifetime and atomically swaps its active backend handler. */
public synchronized void activateBackendPluginMessageHandler(GlobalMessageHandler target) {
- backendPluginMessageTarget.set(target);
if (backendPluginMessageRelay == null) {
backendPluginMessageRelay = new PluginMessageHandler() {
@Override
@@ -1273,8 +1640,16 @@ public void onReceive(com.bencodez.simpleapi.servercomm.codec.JsonEnvelope envel
if (current != null) current.onMessage(envelope);
}
};
- getPluginMessaging().add(backendPluginMessageRelay);
}
+ com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessage current = getPluginMessaging();
+ if (backendPluginMessageRelayOwner != current) {
+ if (backendPluginMessageRelayOwner != null) {
+ backendPluginMessageRelayOwner.getPluginMessages().remove(backendPluginMessageRelay);
+ }
+ current.add(backendPluginMessageRelay);
+ backendPluginMessageRelayOwner = current;
+ }
+ backendPluginMessageTarget.set(target);
}
public void deactivateBackendPluginMessageHandler(GlobalMessageHandler target) {
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java
index 8395c38b4..8847eeebb 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java
@@ -25,6 +25,7 @@
import com.bencodez.votingplugin.backendproxy.transport.BackendProxyTransportManager;
import com.bencodez.votingplugin.backendproxy.voteparty.BackendVotePartySync;
import com.bencodez.votingplugin.proxy.BungeeMethod;
+import com.bencodez.votingplugin.proxy.VotingPluginWire;
import lombok.Getter;
@@ -39,6 +40,11 @@ public class BackendProxyHandler implements Listener {
private final BackendGlobalDataSync globalDataSync;
private BackendPresenceManager presenceManager;
+ private boolean presenceReportingActivated;
+ private final Object inboundPublication = new Object();
+ private boolean inboundPublished;
+ private boolean inboundAborted;
+ private BackendProxyHandler inboundRollbackTarget;
private BackendVotePartySync votePartySync;
private BackendProxyMessageRouter messageRouter;
@@ -62,12 +68,26 @@ public BackendProxyHandler(VotingPluginMain plugin, ProcessedVoteCache processed
* Loads the configured backend/proxy communication components.
*/
public void load() {
+ load(true);
+ }
+
+ /** Loads a replacement without announcing a new presence generation before publication. */
+ public void loadForReplacement() {
+ load(false);
+ }
+
+ private void load(boolean activatePresenceReporting) {
plugin.debug("Loading backend proxy handler");
method = BungeeMethod.getByName(plugin.getBungeeSettings().getBungeeMethod());
plugin.getLogger().info("Using BungeeMethod: " + method.toString());
globalDataSync.load();
globalMessageHandler = new GlobalMessageHandler() {
+ @Override
+ public void onMessage(JsonEnvelope envelope) {
+ BackendProxyHandler.this.dispatchIncomingAfterPublication(envelope, () -> super.onMessage(envelope));
+ }
+
@Override
public void sendMessage(JsonEnvelope envelope) {
transportManager.send(envelope);
@@ -79,20 +99,40 @@ public void sendMessage(JsonEnvelope envelope) {
messageRouter = new BackendProxyMessageRouter(plugin, presenceManager, globalDataSync, votePartySync,
processedVoteCache);
messageRouter.register(globalMessageHandler, method);
- transportManager.start(method, globalMessageHandler);
+ transportManager.start(method, globalMessageHandler, activatePresenceReporting);
if (plugin.getOptions().getServer().equalsIgnoreCase("pleaseset")) {
plugin.getLogger().warning("Server name for bungee voting is not set, please set it");
}
- presenceManager.start();
+ if (activatePresenceReporting) {
+ activatePresenceReporting();
+ activateInboundMessages();
+ }
+ }
+
+ /** Starts presence only after a staged handler reaches the atomic publication boundary. */
+ public void activatePresenceReporting() {
+ if (presenceManager != null && !presenceReportingActivated) {
+ presenceManager.start();
+ presenceReportingActivated = true;
+ }
+ // Presence startup can throw while scheduling its heartbeat. Keep inbound
+ // HTTP callbacks behind the publication barrier until every fallible part of
+ // the replacement is active, so rollback cannot race a queued callback.
+ transportManager.activateAfterPublication();
}
/**
* Closes backend/proxy components and persists cached proxy state.
*/
public void close() {
- if (presenceManager != null) {
+ synchronized (inboundPublication) {
+ if (!inboundPublished && !inboundAborted) inboundAborted = true;
+ inboundPublication.notifyAll();
+ }
+ if (presenceManager != null && presenceReportingActivated) {
presenceManager.stop();
+ presenceReportingActivated = false;
}
transportManager.close();
if (votePartySync != null) {
@@ -101,26 +141,207 @@ public void close() {
globalDataSync.close();
}
- /** Releases a same-method subscriber/listener before its replacement starts. */
- public void prepareForReplacement(BungeeMethod replacementMethod) {
- if (method == replacementMethod && method != BungeeMethod.PLUGINMESSAGING && method != BungeeMethod.REDIS) {
+ /** Opens inbound dispatch only after the replacement and all handoffs are committed. */
+ public void activateInboundMessages() {
+ synchronized (inboundPublication) {
+ if (inboundAborted) return;
+ inboundPublished = true;
+ inboundPublication.notifyAll();
+ }
+ }
+
+ /** Routes an already accepted staged callback through the restored predecessor on rollback. */
+ public void abortStagedInboundTo(BackendProxyHandler previous) {
+ synchronized (inboundPublication) {
+ if (inboundPublished || inboundAborted) return;
+ inboundRollbackTarget = previous;
+ inboundAborted = true;
+ inboundPublication.notifyAll();
+ }
+ }
+
+ void dispatchIncomingAfterPublication(JsonEnvelope envelope, Runnable localDispatch) {
+ BackendProxyHandler rollbackTarget;
+ synchronized (inboundPublication) {
+ while (!inboundPublished && !inboundAborted) {
+ try {
+ inboundPublication.wait();
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ return;
+ }
+ }
+ rollbackTarget = inboundPublished ? null : inboundRollbackTarget;
+ if (!inboundPublished && rollbackTarget == null) return;
+ }
+ if (rollbackTarget != null) {
+ GlobalMessageHandler rollbackHandler = rollbackTarget.globalMessageHandler;
+ if (rollbackHandler != null) rollbackHandler.onMessage(envelope);
+ return;
+ }
+ // Global-data checks perform synchronous SQL and already run on their own
+ // timer worker. Explicitly dispatch an inbound wake-up asynchronously too:
+ // plugin messaging can invoke this method on Bukkit's primary thread.
+ if (VotingPluginWire.SUB_BUNGEE_TIME_CHANGE.equals(envelope.getSubChannel())) {
+ plugin.getBukkitScheduler().runTaskAsynchronously(plugin, localDispatch);
+ return;
+ }
+ plugin.getBukkitScheduler().executeOrScheduleSync(plugin, localDispatch);
+ }
+
+ /** Returns whether replacement preparation must preserve accepted deliveries. */
+ public boolean requiresPreparationForReplacement() {
+ return method == BungeeMethod.HTTP || method == BungeeMethod.PLUGINMESSAGING
+ || transportManager.hasPendingAsyncHandoff() || transportManager.hasPendingRedisReplay();
+ }
+
+ /** Prepares HTTP state or waits off-thread for an earlier cross-transport handoff. */
+ public boolean prepareForReplacement(BungeeMethod replacementMethod) {
+ return prepareForReplacement(replacementMethod,
+ System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(25));
+ }
+
+ public boolean prepareForReplacement(BungeeMethod replacementMethod, long deadlineNanos) {
+ // SocketHandler swallows a listener bind failure. A same-method replacement
+ // must therefore retire its listener before the staged handler is started;
+ // rollback recreates this prepared transport if validation later fails.
+ if (method == BungeeMethod.SOCKETS && replacementMethod == BungeeMethod.SOCKETS) {
+ transportManager.prepareForReplacement();
+ return true;
+ }
+ if (method == BungeeMethod.MQTT && replacementMethod == BungeeMethod.MQTT) {
+ // A duplicate MQTT ClientID disconnects the live broker session, so stage
+ // only after retiring the predecessor and restore it on validation rollback.
+ transportManager.prepareForReplacement();
+ return true;
+ }
+ if (method == BungeeMethod.HTTP) {
transportManager.prepareForReplacement();
+ return true;
}
+ if (method == BungeeMethod.PLUGINMESSAGING) {
+ transportManager.prepareAsyncHandoffForReplacement(deadlineNanos);
+ return true;
+ }
+ if (method == BungeeMethod.REDIS && replacementMethod != BungeeMethod.REDIS) {
+ if (transportManager.hasPendingAsyncHandoff())
+ transportManager.prepareAsyncHandoffForReplacement(deadlineNanos);
+ return transportManager.prepareRedisReplayTransition(replacementMethod, deadlineNanos);
+ }
+ if (method == BungeeMethod.REDIS) {
+ // Same-Redis retirement installs its send fence during the bounded
+ // off-thread handoff. Returning true here lets the staged replacement
+ // buffer its own sends until the predecessor FIFO is admitted at Bukkit
+ // publication.
+ return true;
+ }
+ if (!transportManager.hasPendingAsyncHandoff()) return false;
+ transportManager.prepareAsyncHandoffForReplacement(deadlineNanos);
+ return true;
+ }
+
+ /** Atomically fences new sends only when disabling cannot discard prepared HTTP messages. */
+ public boolean commitPreparedDisable() {
+ return transportManager.commitPreparedDisable();
+ }
+
+ /** Publishes the final presence update while fencing other sends before transport preparation. */
+ public void preparePresenceForDisable() {
+ preparePresenceForDisableInternal(null);
+ }
+
+ /** Publishes the final presence update before the caller's validation deadline. */
+ public void preparePresenceForDisable(long deadlineNanos) {
+ preparePresenceForDisableInternal(deadlineNanos);
+ }
+
+ private void preparePresenceForDisableInternal(Long deadlineNanos) {
+ transportManager.beginPreparedDisable();
+ if (presenceManager != null && presenceReportingActivated) {
+ // stopForDisable may reject when the transport cannot accept the final
+ // presence update. Mark this inactive first so rollback can start it again.
+ presenceReportingActivated = false;
+ if (deadlineNanos == null) presenceManager.stopForDisable();
+ else presenceManager.stopForDisable(deadlineNanos);
+ }
+ }
+
+ /** Restores delivery and presence when a prepared disable is abandoned. */
+ public void restorePresenceAfterFailedDisablePreparation() {
+ transportManager.cancelPreparedDisable();
+ activatePresenceReporting();
+ }
+
+ public void beginPreparedHttpHandoff() {
+ transportManager.beginPreparedHttpHandoff();
+ }
+
+ /** Reserves staged HTTP capacity before publication can admit replacement sends. */
+ public void reservePreparedHttpHandoff(BackendProxyHandler replacement) {
+ transportManager.reservePreparedTransportHandoff(replacement.transportManager);
+ }
+
+ /** Restores a prepared HTTP transport when its replacement fails validation. */
+ public void restoreAfterFailedReplacement() {
+ transportManager.restoreAfterFailedReplacement();
+ }
+
+ /** Restores a failed same-Redis predecessor without discarding its replacement's replay FIFO. */
+ public void restoreAfterFailedReplacement(BackendProxyHandler failedReplacement) {
+ transportManager.restoreAfterFailedReplacement(
+ failedReplacement == null ? null : failedReplacement.transportManager);
+ }
+
+ /** Reasserts the old handler with a fresh presence generation after rollback. */
+ public void refreshPresenceAfterFailedReplacement() {
+ if (presenceManager != null && presenceReportingActivated) {
+ presenceManager.stop();
+ presenceManager.start();
+ }
+ }
+
+ public void awaitRestoreAfterFailedReplacement(long deadlineNanos) {
+ transportManager.awaitPreparedTransportRestoration(deadlineNanos);
}
/** Fails a configuration apply when its selected transport did not initialize. */
public void validateTransport() {
+ validateTransport(System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(25));
+ }
+
+ /** Validates transport startup without extending the caller's existing deadline. */
+ public void validateTransport(long deadlineNanos) {
if (method == null || globalMessageHandler == null || presenceManager == null) {
throw new IllegalStateException("Backend proxy handler initialization failed");
}
- transportManager.validate();
+ transportManager.validate(deadlineNanos);
}
/** Completes the no-loss/no-duplicate same-Redis subscriber handoff after validation. */
public void completeRedisHandoff(BackendProxyHandler replacement) {
- if (method != BungeeMethod.REDIS || replacement.method != BungeeMethod.REDIS) return;
- transportManager.closeRedisForHandoff();
- replacement.transportManager.activateRedisAfterHandoff();
+ if (!requiresRedisHandoff(replacement)) return;
+ transportManager.completeRedisHandoff(replacement.transportManager);
+ }
+
+ /** Returns whether this handler owns a Redis listener whose shutdown can block. */
+ public boolean requiresRedisRetirement() {
+ return method == BungeeMethod.REDIS;
+ }
+
+ /** Returns whether this replacement needs the bounded same-Redis retirement path. */
+ public boolean requiresRedisHandoff(BackendProxyHandler replacement) {
+ return replacement != null && method == BungeeMethod.REDIS && replacement.method == BungeeMethod.REDIS;
+ }
+
+ /** Replays Redis handoff deliveries only after inbound publication is open. */
+ public void replayRedisAfterHandoffPublication() {
+ transportManager.replayRedisAfterHandoffPublication();
+ }
+
+ /** Forwards messages buffered while the previous transport was fenced. */
+ public void completeHttpHandoff(BackendProxyHandler replacement) {
+ if (replacement == null) return;
+ transportManager.completePreparedTransportHandoff(replacement.transportManager);
}
public void playerOnline(String playerName, String uuid) {
@@ -136,14 +357,15 @@ public void playerOffline(String playerName) {
}
public void reloadPresenceReporting() {
- if (presenceManager != null) {
+ if (presenceManager != null && presenceReportingActivated) {
presenceManager.reload();
}
}
public void disablePresenceReporting() {
- if (presenceManager != null) {
+ if (presenceManager != null && presenceReportingActivated) {
presenceManager.stop();
+ presenceReportingActivated = false;
}
}
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java
index 919fb8077..5029b7e4d 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java
@@ -25,6 +25,7 @@ public class ProcessedVoteCache {
private final LinkedHashMap processedRedisDeliveries = new LinkedHashMap<>();
private final LinkedHashMap legacyRedisDeliveries = new LinkedHashMap<>();
private long legacyRedisDeliveryBytes;
+ private boolean legacyRedisHandoffOverflowed;
private Object activeRedisSubscriber;
private Object standbyRedisSubscriber;
@@ -82,14 +83,18 @@ public synchronized boolean reserveRedisDelivery(String deliveryId) {
return true;
}
- public synchronized void registerRedisSubscriber(Object subscriber) {
+ public synchronized boolean registerRedisSubscriber(Object subscriber) {
if (activeRedisSubscriber == null) {
activeRedisSubscriber = subscriber;
+ return true;
} else if (activeRedisSubscriber != subscriber) {
standbyRedisSubscriber = subscriber;
legacyRedisDeliveries.clear();
legacyRedisDeliveryBytes = 0;
+ legacyRedisHandoffOverflowed = false;
+ return false;
}
+ return true;
}
/** Returns true only for the active subscriber and counts its legacy delivery during overlap. */
@@ -107,12 +112,16 @@ public synchronized boolean reserveLegacyRedisDelivery(Object subscriber, String
&& legacyRedisDeliveryBytes <= MAX_LEGACY_REDIS_TOTAL_BYTES - bytes) {
legacyRedisDeliveries.put(signature, 1);
legacyRedisDeliveryBytes += bytes;
- }
+ } else legacyRedisHandoffOverflowed = true;
}
}
return true;
}
+ public synchronized boolean isLegacyRedisHandoffOverflowed() {
+ return legacyRedisHandoffOverflowed;
+ }
+
public synchronized void activateRedisSubscriber(Object subscriber) {
if (standbyRedisSubscriber != subscriber) {
throw new IllegalStateException("Redis replacement subscriber is not registered");
@@ -121,6 +130,15 @@ public synchronized void activateRedisSubscriber(Object subscriber) {
standbyRedisSubscriber = null;
}
+ /** Restores the retired active listener after a promoted replacement is rolled back. */
+ public synchronized void restoreRedisSubscriber(Object subscriber) {
+ activeRedisSubscriber = subscriber;
+ standbyRedisSubscriber = null;
+ legacyRedisDeliveries.clear();
+ legacyRedisDeliveryBytes = 0;
+ legacyRedisHandoffOverflowed = false;
+ }
+
/** Consumes one matching delivery processed by the previous active subscriber. */
public synchronized boolean consumeLegacyRedisDelivery(String signature) {
Integer count = legacyRedisDeliveries.get(signature);
@@ -135,6 +153,7 @@ public synchronized boolean consumeLegacyRedisDelivery(String signature) {
public synchronized void finishRedisHandoff() {
legacyRedisDeliveries.clear();
legacyRedisDeliveryBytes = 0;
+ legacyRedisHandoffOverflowed = false;
}
public synchronized void unregisterRedisSubscriber(Object subscriber) {
@@ -142,6 +161,7 @@ public synchronized void unregisterRedisSubscriber(Object subscriber) {
standbyRedisSubscriber = null;
legacyRedisDeliveries.clear();
legacyRedisDeliveryBytes = 0;
+ legacyRedisHandoffOverflowed = false;
}
if (activeRedisSubscriber == subscriber) activeRedisSubscriber = null;
}
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSync.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSync.java
index 29c24a7db..febe99c28 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSync.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSync.java
@@ -4,9 +4,12 @@
import java.time.ZoneOffset;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
import com.bencodez.advancedcore.api.time.TimeType;
@@ -29,6 +32,8 @@ public class BackendGlobalDataSync {
private final VotingPluginMain plugin;
private final Consumer sender;
+ private final AtomicBoolean forceUpdateInProgress = new AtomicBoolean(false);
+ private final Set timeChangesInProgress = ConcurrentHashMap.newKeySet();
@Getter
private GlobalDataHandler globalDataHandler;
@@ -46,26 +51,40 @@ public void checkGlobalData() {
}
HashMap data = globalDataHandler.getExact(plugin.getBungeeSettings().getServer());
- if (data.containsKey("ForceUpdate") && checkGlobalDataTimeValue(data.get("ForceUpdate"))) {
- if (plugin.getStorageType().equals(UserStorage.MYSQL)) {
- plugin.getMysql().clearCacheBasic();
+ if (data.containsKey("ForceUpdate") && checkGlobalDataTimeValue(data.get("ForceUpdate"))
+ && forceUpdateInProgress.compareAndSet(false, true)) {
+ String serverName = plugin.getBungeeSettings().getServer();
+ try {
+ if (UserStorage.MYSQL.equals(plugin.getStorageType())) {
+ plugin.getMysql().clearCacheBasic();
+ }
+ plugin.getBukkitScheduler().executeOrScheduleSync(plugin, () -> {
+ try {
+ plugin.getUserManager().getDataManager().clearCache();
+ plugin.setUpdate(true);
+ plugin.update();
+ } catch (RuntimeException failure) {
+ forceUpdateInProgress.set(false);
+ plugin.debug(failure);
+ return;
+ }
+ try {
+ plugin.getBukkitScheduler().runTaskAsynchronously(plugin,
+ () -> clearForceUpdateFlag(serverName));
+ } catch (RuntimeException failure) {
+ forceUpdateInProgress.set(false);
+ plugin.debug(failure);
+ }
+ });
+ } catch (RuntimeException failure) {
+ forceUpdateInProgress.set(false);
+ plugin.debug(failure);
}
- plugin.getUserManager().getDataManager().clearCache();
- plugin.setUpdate(true);
- plugin.update();
- globalDataHandler.setBoolean(plugin.getBungeeSettings().getServer(), "ForceUpdate", false);
}
- boolean forceUpdate = checkGlobalDataTime(TimeType.MONTH, data);
- forceUpdate |= checkGlobalDataTime(TimeType.WEEK, data);
- forceUpdate |= checkGlobalDataTime(TimeType.DAY, data);
-
- if (forceUpdate) {
- HashMap dataToSet = new HashMap<>();
- dataToSet.put("FinishedProcessing", new DataValueBoolean(true));
- dataToSet.put("Processing", new DataValueBoolean(false));
- globalDataHandler.setData(plugin.getBungeeSettings().getServer(), dataToSet);
- }
+ checkGlobalDataTime(TimeType.MONTH, data);
+ checkGlobalDataTime(TimeType.WEEK, data);
+ checkGlobalDataTime(TimeType.DAY, data);
}
public boolean checkGlobalDataTime(TimeType type, HashMap data) {
@@ -80,18 +99,58 @@ public boolean checkGlobalDataTime(TimeType type, HashMap dat
globalDataHandler.setBoolean(plugin.getBungeeSettings().getServer(), type.toString(), false);
return false;
}
+ if (!timeChangesInProgress.add(type)) {
+ return false;
+ }
- globalDataHandler.setBoolean(plugin.getBungeeSettings().getServer(), "Processing", true);
+ String serverName = plugin.getBungeeSettings().getServer();
+ globalDataHandler.setBoolean(serverName, "Processing", true);
plugin.debug("Detected time change from bungee: " + type.toString());
- plugin.getTimeChecker().forceChanged(type, false, true, true);
- globalDataHandler.setBoolean(plugin.getBungeeSettings().getServer(), type.toString(), false);
-
- JsonEnvelope.Builder builder = JsonEnvelope.builder("TimeChangeFinished").schema(VotingPluginWire.SCHEMA_VERSION);
- builder.put("server", plugin.getBungeeSettings().getServer());
- sender.accept(builder.build());
+ try {
+ plugin.getBukkitScheduler().executeOrScheduleSync(plugin, () -> {
+ try {
+ plugin.getTimeChecker().forceChanged(type, false, true, true);
+ } catch (RuntimeException failure) {
+ timeChangesInProgress.remove(type);
+ plugin.debug(failure);
+ return;
+ }
+ try {
+ plugin.getBukkitScheduler().runTaskAsynchronously(plugin,
+ () -> finishTimeChange(type, serverName));
+ } catch (RuntimeException failure) {
+ timeChangesInProgress.remove(type);
+ plugin.debug(failure);
+ }
+ });
+ } catch (RuntimeException failure) {
+ timeChangesInProgress.remove(type);
+ plugin.debug(failure);
+ return false;
+ }
return true;
}
+ private void finishTimeChange(TimeType type, String serverName) {
+ boolean completed = false;
+ try {
+ globalDataHandler.setBoolean(serverName, type.toString(), false);
+ JsonEnvelope.Builder builder = JsonEnvelope.builder("TimeChangeFinished")
+ .schema(VotingPluginWire.SCHEMA_VERSION);
+ builder.put("server", serverName);
+ sender.accept(builder.build());
+ completed = true;
+ } finally {
+ timeChangesInProgress.remove(type);
+ if (completed && timeChangesInProgress.isEmpty()) {
+ HashMap dataToSet = new HashMap<>();
+ dataToSet.put("FinishedProcessing", new DataValueBoolean(true));
+ dataToSet.put("Processing", new DataValueBoolean(false));
+ globalDataHandler.setData(serverName, dataToSet);
+ }
+ }
+ }
+
public boolean checkGlobalDataTimeValue(DataValue data) {
if (data.isBoolean()) {
return data.getBoolean();
@@ -99,6 +158,14 @@ public boolean checkGlobalDataTimeValue(DataValue data) {
return Boolean.valueOf(data.getString());
}
+ private void clearForceUpdateFlag(String serverName) {
+ try {
+ globalDataHandler.setBoolean(serverName, "ForceUpdate", false);
+ } finally {
+ forceUpdateInProgress.set(false);
+ }
+ }
+
public void load() {
if (!plugin.getBungeeSettings().isGloblalDataEnabled()) {
return;
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java
index a82c5551a..91c601d81 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java
@@ -3,9 +3,12 @@
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import org.bukkit.Bukkit;
import org.bukkit.entity.Player;
@@ -34,6 +37,8 @@ public class BackendPresenceManager {
private final Object lifecycleLock = new Object();
private boolean reporting;
+ /** Invalidates delayed lifecycle callbacks when a fresh incarnation starts. */
+ private long lifecycleGeneration;
private String server;
private UUID incarnationId;
private long startedAt;
@@ -57,6 +62,7 @@ public void start() {
}
String configuredServer = plugin.getBungeeSettings().getServer();
synchronized (lifecycleLock) {
+ lifecycleGeneration++;
long now = System.currentTimeMillis();
incarnationId = UUID.randomUUID();
startedAt = now;
@@ -67,24 +73,90 @@ public void start() {
lastResyncRequestAtNanos = 0L;
lastSnapshotRequestId = null;
lastSnapshotRequestAtNanos = 0L;
- send(VotingPluginWire.backendStarted(server, incarnationId, startedAt, now));
- send(VotingPluginWire.backendHeartbeat(server, incarnationId, startedAt, nextTimestamp()));
-
if (heartbeatTask != null) {
heartbeatTask.cancel(false);
}
- heartbeatTask = plugin.getTimer().scheduleAtFixedRate(new Runnable() {
- @Override
- public void run() {
- sendHeartbeat();
- }
- }, HEARTBEAT_SECONDS, HEARTBEAT_SECONDS, TimeUnit.SECONDS);
+ try {
+ heartbeatTask = plugin.getTimer().scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ sendHeartbeat();
+ }
+ }, HEARTBEAT_SECONDS, HEARTBEAT_SECONDS, TimeUnit.SECONDS);
+ seedOnlinePlayers();
+ } catch (RuntimeException failure) {
+ if (heartbeatTask != null) heartbeatTask.cancel(false);
+ heartbeatTask = null;
+ reporting = false;
+ server = null;
+ incarnationId = null;
+ throw failure;
+ }
+ send(VotingPluginWire.backendStarted(server, incarnationId, startedAt, now));
+ send(VotingPluginWire.backendHeartbeat(server, incarnationId, startedAt, nextTimestamp()));
}
- seedOnlinePlayers();
}
public void stop() {
+ stop(false);
+ }
+
+ /** Stops presence and propagates rejection when a configuration disable must be transactional. */
+ public void stopForDisable() {
+ stopForDisable(System.nanoTime() + TimeUnit.SECONDS.toNanos(5));
+ }
+
+ /** Stops presence before the caller's transactional validation deadline. */
+ public void stopForDisable(long deadlineNanos) {
+ // Plugin-message transport ultimately calls Bukkit's sendPluginMessage API,
+ // which is primary-thread-only. Control validation runs on its worker, so
+ // marshal the transactional stopped-presence send before returning the
+ // result to that worker. Unit-test and shutdown contexts without a server
+ // retain the direct path used by the non-Control lifecycle.
+ if (method == BungeeMethod.PLUGINMESSAGING && plugin != null && plugin.getServer() != null
+ && !plugin.getServer().isPrimaryThread()) {
+ long expectedGeneration;
+ synchronized (lifecycleLock) {
+ expectedGeneration = lifecycleGeneration;
+ }
+ CompletableFuture scheduled = new CompletableFuture<>();
+ try {
+ plugin.getBukkitScheduler().executeOrScheduleSync(plugin, () -> {
+ try {
+ stop(true, expectedGeneration);
+ scheduled.complete(null);
+ } catch (Throwable failure) {
+ scheduled.completeExceptionally(failure);
+ }
+ });
+ long remaining = deadlineNanos - System.nanoTime();
+ if (remaining <= 0L) throw new TimeoutException("Backend stopped presence deadline expired");
+ scheduled.get(remaining, TimeUnit.NANOSECONDS);
+ return;
+ } catch (Exception failure) {
+ scheduled.cancel(false);
+ Throwable cause = failure instanceof ExecutionException && failure.getCause() != null
+ ? failure.getCause() : failure;
+ if (cause instanceof RuntimeException runtime) throw runtime;
+ throw new IllegalStateException("Backend stopped presence could not run on the Bukkit thread", cause);
+ }
+ }
+ stop(true);
+ }
+
+ private void stop(boolean requireStoppedDelivery) {
+ stop(requireStoppedDelivery, -1L);
+ }
+
+ /**
+ * Stops only the generation captured by a delayed callback. A cancelled future
+ * does not necessarily cancel a task already queued on the Bukkit scheduler.
+ */
+ private void stop(boolean requireStoppedDelivery, long expectedGeneration) {
synchronized (lifecycleLock) {
+ if (expectedGeneration >= 0L && lifecycleGeneration != expectedGeneration) {
+ return;
+ }
String activeServer = server;
UUID activeIncarnationId = incarnationId;
long activeStartedAt = startedAt;
@@ -95,16 +167,21 @@ public void stop() {
heartbeatTask.cancel(false);
heartbeatTask = null;
}
- if (wasReporting && activeServer != null && activeIncarnationId != null) {
- send(VotingPluginWire.backendStopped(activeServer, activeIncarnationId, activeStartedAt,
- nextTimestamp()));
+ try {
+ if (wasReporting && activeServer != null && activeIncarnationId != null) {
+ JsonEnvelope stopped = VotingPluginWire.backendStopped(activeServer, activeIncarnationId,
+ activeStartedAt, nextTimestamp());
+ if (requireStoppedDelivery) globalMessageHandler.sendMessage(stopped);
+ else send(stopped);
+ }
+ } finally {
+ incarnationId = null;
+ lastResyncRequestId = null;
+ lastResyncRequestAtNanos = 0L;
+ lastSnapshotRequestId = null;
+ lastSnapshotRequestAtNanos = 0L;
+ playerSessions.clear();
}
- incarnationId = null;
- lastResyncRequestId = null;
- lastResyncRequestAtNanos = 0L;
- lastSnapshotRequestId = null;
- lastSnapshotRequestAtNanos = 0L;
- playerSessions.clear();
}
}
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransport.java
index 30a0baa00..ab4a7f6ae 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransport.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransport.java
@@ -10,7 +10,17 @@ public interface BackendProxyTransport {
void start(GlobalMessageHandler messageHandler);
- void send(JsonEnvelope envelope);
+ /**
+ * Starts the transport, optionally allowing startup-only transient failures to
+ * be retried. Replacement transports disable this so Control validation stays
+ * fail-fast; ordinary startup retains its recovery loop.
+ */
+ default void start(GlobalMessageHandler messageHandler, boolean retryInitialization) {
+ start(messageHandler);
+ }
+
+ /** Returns true only when the delivery was accepted by the transport or its durable queue. */
+ boolean send(JsonEnvelope envelope);
default void validate() {
}
@@ -19,5 +29,9 @@ default void prepareForReplacement() {
close();
}
+ /** Activates transport state that must not become visible before handler publication. */
+ default void activateAfterPublication() {
+ }
+
void close();
}
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java
index 4d7b20872..939ea2047 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java
@@ -10,15 +10,37 @@
import com.bencodez.votingplugin.VotingPluginMain;
import com.bencodez.votingplugin.backendproxy.cache.ProcessedVoteCache;
import com.bencodez.votingplugin.proxy.BungeeMethod;
+import com.bencodez.votingplugin.proxy.VotingPluginWire;
/**
* Selects and owns the active backend-to-proxy transport.
*/
public class BackendProxyTransportManager {
+ private static final int MAX_PREPARED_SENDS = 1024;
+ private static final int MAX_ASYNC_HANDOFF_SENDS = 6144;
+ private static final int PLUGIN_MESSAGE_HANDOFF_BATCH_SIZE = 32;
private final VotingPluginMain plugin;
private final ProcessedVoteCache processedVoteCache;
private BackendProxyTransport transport;
+ private BackendProxyTransport preparedTransport;
+ private BackendProxyTransport retiredTransport;
+ // The old Redis instance is already stopped after a successful worker-side
+ // handoff, but keeps its captured handler long enough to restore it if Bukkit
+ // publication subsequently fails or is abandoned.
+ private RedisBackendProxyTransport completedRedisHandoffTransport;
+ private BackendProxyTransportManager forwardingManager;
+ private final java.util.ArrayDeque preparedSends = new java.util.ArrayDeque<>();
+ private final java.util.ArrayDeque asyncHandoffSends = new java.util.ArrayDeque<>();
+ private Thread asyncHandoffWorker;
+ private boolean pluginMessageHandoffScheduled;
+ private long handoffGeneration;
+ private boolean preparedSendFence;
+ private boolean rejectPreparedSends;
+ private boolean allowStoppedPresenceDuringDisable;
+ private boolean preparedQueueWarning;
+ private boolean rejectedSendWarning;
+ private boolean asyncHandoffRetryWarning;
public BackendProxyTransportManager(VotingPluginMain plugin) {
this(plugin, new ProcessedVoteCache());
@@ -30,6 +52,10 @@ public BackendProxyTransportManager(VotingPluginMain plugin, ProcessedVoteCache
}
public void start(BungeeMethod method, GlobalMessageHandler messageHandler) {
+ start(method, messageHandler, true);
+ }
+
+ public void start(BungeeMethod method, GlobalMessageHandler messageHandler, boolean retryInitialization) {
close();
switch (method) {
case MYSQL:
@@ -41,6 +67,9 @@ public void start(BungeeMethod method, GlobalMessageHandler messageHandler) {
case SOCKETS:
transport = new SocketBackendProxyTransport(plugin);
break;
+ case HTTP:
+ transport = new HttpBackendProxyTransport(plugin);
+ break;
case REDIS:
transport = new RedisBackendProxyTransport(plugin, processedVoteCache);
break;
@@ -50,40 +79,585 @@ public void start(BungeeMethod method, GlobalMessageHandler messageHandler) {
default:
throw new IllegalArgumentException("Unsupported backend proxy method: " + method);
}
- transport.start(messageHandler);
+ transport.start(messageHandler, retryInitialization);
}
- public void send(JsonEnvelope envelope) {
- if (transport != null) {
+ public synchronized void send(JsonEnvelope envelope) {
+ if (rejectPreparedSends) {
+ if (allowStoppedPresenceDuringDisable && envelope != null
+ && VotingPluginWire.SUB_BACKEND_STOPPED.equals(envelope.getSubChannel())) {
+ allowStoppedPresenceDuringDisable = false;
+ BackendProxyTransport stoppingTransport = transport != null ? transport : preparedTransport;
+ if (stoppingTransport == null || !stoppingTransport.send(envelope))
+ throw new IllegalStateException("Backend stopped presence was not accepted before disabling transport");
+ return;
+ }
+ if (!rejectedSendWarning) {
+ rejectedSendWarning = true;
+ plugin.getLogger().severe("Backend proxy transport is disabled; delivery was not accepted");
+ }
+ } else if (preparedSendFence) {
+ acceptPreparedSend(envelope);
+ } else if (forwardingManager != null) {
+ forwardingManager.send(envelope);
+ } else if (hasPendingAsyncHandoff()) {
+ acceptQueuedTransportSend(envelope);
+ } else if (transport instanceof PluginMessagingBackendProxyTransport) {
+ acceptQueuedTransportSend(envelope);
+ } else if (transport != null) {
transport.send(envelope);
+ } else if (preparedTransport != null) {
+ acceptPreparedSend(envelope);
+ }
+ }
+
+ private void acceptPreparedSend(JsonEnvelope envelope) {
+ if (preparedSends.size() < MAX_PREPARED_SENDS) {
+ preparedSends.addLast(envelope);
+ } else if (!preparedQueueWarning) {
+ preparedQueueWarning = true;
+ plugin.getLogger().severe("Backend proxy replacement handoff queue is full; delivery was not accepted");
}
}
- public void close() {
+ public void activateAfterPublication() {
+ if (transport != null) transport.activateAfterPublication();
+ }
+
+ public synchronized void close() {
+ handoffGeneration++;
+ if (asyncHandoffWorker != null) asyncHandoffWorker.interrupt();
+ asyncHandoffWorker = null;
+ pluginMessageHandoffScheduled = false;
+ asyncHandoffSends.clear();
+ notifyAll();
if (transport != null) {
transport.close();
transport = null;
}
+ if (preparedTransport != null) {
+ preparedTransport.close();
+ preparedTransport = null;
+ }
+ if (retiredTransport != null) {
+ BackendProxyTransport retired = retiredTransport;
+ if (retired instanceof RedisBackendProxyTransport redis) {
+ // A failed Redis listener shutdown can spend its bounded join timeout in
+ // close(). Publication calls the predecessor's close on Bukkit, so retry
+ // that fenced cleanup independently instead of stalling publication.
+ retiredTransport = null;
+ closeRetiredRedisAsync(redis);
+ } else try {
+ retired.close();
+ retiredTransport = null;
+ } catch (RuntimeException cleanupFailure) {
+ // This transport has already been fenced and replaced. Keep its handle
+ // for another cleanup attempt, but never fail or close the live replacement.
+ if (plugin != null) {
+ plugin.getLogger().warning("Retired backend proxy transport did not stop cleanly");
+ plugin.debug(cleanupFailure);
+ }
+ }
+ }
+ completedRedisHandoffTransport = null;
+ if (forwardingManager == null) preparedSends.clear();
+ }
+
+ /** Retries only a fenced retired Redis listener off the Bukkit publication callback. */
+ private void closeRetiredRedisAsync(RedisBackendProxyTransport retired) {
+ Thread cleanup = new Thread(() -> {
+ RuntimeException cleanupFailure = null;
+ for (int attempt = 0; attempt < 3; attempt++) {
+ try {
+ retired.close();
+ return;
+ } catch (RuntimeException failure) {
+ cleanupFailure = failure;
+ if (attempt == 2) break;
+ try {
+ Thread.sleep(250L);
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ break;
+ }
+ }
+ }
+ if (cleanupFailure != null && plugin != null) {
+ plugin.getLogger().warning("Retired Redis backend listener did not stop cleanly during async cleanup");
+ plugin.debug(cleanupFailure);
+ }
+ }, "VotingPlugin-Retired-Redis-Cleanup");
+ cleanup.setDaemon(true);
+ cleanup.start();
}
public void validate() {
+ validate(System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(25));
+ }
+
+ public void validate(long deadlineNanos) {
if (transport == null) throw new IllegalStateException("Backend proxy transport was not initialized");
- transport.validate();
+ if (transport instanceof HttpBackendProxyTransport http) http.validate(deadlineNanos);
+ else transport.validate();
}
public void prepareForReplacement() {
- if (transport != null) {
- transport.prepareForReplacement();
+ BackendProxyTransport candidate;
+ synchronized (this) {
+ if (transport == null) return;
+ candidate = transport;
+ preparedTransport = candidate;
transport = null;
}
+ try {
+ // HTTP preparation can wait on a bounded network flush. Keep send() free to
+ // enqueue presence and vote messages while that I/O is in progress.
+ candidate.prepareForReplacement();
+ } catch (RuntimeException failure) {
+ synchronized (this) {
+ if (preparedTransport != candidate) throw failure;
+ HttpBackendProxyTransport http = candidate instanceof HttpBackendProxyTransport prepared
+ ? prepared : null;
+ MqttBackendProxyTransport mqtt = candidate instanceof MqttBackendProxyTransport prepared
+ ? prepared : null;
+ if ((http != null && !http.isClosedForReplacement()) || (mqtt != null && mqtt.isConnected())) {
+ // A failed flush deliberately restarts the existing connector. Reinstall
+ // that live instance instead of creating a second directory owner/client.
+ transport = candidate;
+ preparedTransport = null;
+ while (!preparedSends.isEmpty() && transport.send(preparedSends.peekFirst()))
+ preparedSends.removeFirst();
+ preparedQueueWarning = false;
+ } else {
+ // Enrollment cancellation may already have closed this instance. Restore
+ // from its captured configuration before configuration rollback.
+ try {
+ restorePreparedTransport();
+ } catch (RuntimeException restorationFailure) {
+ failure.addSuppressed(restorationFailure);
+ }
+ }
+ }
+ throw failure;
+ }
}
- public void closeRedisForHandoff() {
- if (!(transport instanceof RedisBackendProxyTransport)) {
- throw new IllegalStateException("Redis backend proxy transport is unavailable");
+ public synchronized void completePreparedTransportHandoff(BackendProxyTransportManager replacement) {
+ if (preparedTransport == null && !preparedSendFence) return;
+ BackendProxyTransportManager target = java.util.Objects.requireNonNull(replacement, "replacement");
+ java.util.ArrayList pending = new java.util.ArrayList<>();
+ if (preparedTransport instanceof HttpBackendProxyTransport http) {
+ pending.addAll(http.preparedMessagesSnapshot());
}
- ((RedisBackendProxyTransport) transport).closeForHandoff();
- transport = null;
+ pending.addAll(preparedSends);
+ target.acceptPreparedHandoffMessages(pending);
+ // Do not consume the old queues or forward subsequent sends until the
+ // replacement has admitted every snapshot. A failed admission therefore
+ // leaves rollback with the complete original FIFO intact.
+ if (preparedTransport instanceof HttpBackendProxyTransport http) http.drainPreparedMessages();
+ preparedSends.clear();
+ forwardingManager = target;
+ preparedSendFence = false;
+ }
+
+ /** Prevents disabling from discarding a delivery accepted during HTTP preparation. */
+ public synchronized boolean commitPreparedDisable() {
+ if (preparedTransport instanceof HttpBackendProxyTransport http && http.preparedMessageCount() != 0) return false;
+ if (!preparedSends.isEmpty()) return false;
+ rejectPreparedSends = true;
+ return true;
+ }
+
+ /** Fences ordinary sends while allowing one final stopped-presence envelope. */
+ public synchronized void beginPreparedDisable() {
+ rejectPreparedSends = true;
+ allowStoppedPresenceDuringDisable = true;
+ }
+
+ /** Reopens delivery when a prepared disable is rolled back. */
+ public synchronized void cancelPreparedDisable() {
+ rejectPreparedSends = false;
+ allowStoppedPresenceDuringDisable = false;
+ rejectedSendWarning = false;
+ }
+
+ public synchronized boolean hasPendingAsyncHandoff() {
+ return !asyncHandoffSends.isEmpty() || asyncHandoffWorker != null || pluginMessageHandoffScheduled;
+ }
+
+ /** Drains an older handoff, then atomically buffers sends until publication. */
+ public synchronized void prepareAsyncHandoffForReplacement(long deadlineNanos) {
+ awaitAsyncHandoff(deadlineNanos);
+ preparedSendFence = true;
+ }
+
+ /** Waits off-thread so a later replacement cannot clear an unfinished admitted handoff. */
+ public synchronized void awaitAsyncHandoff(long deadlineNanos) {
+ while (hasPendingAsyncHandoff()) {
+ long remaining = deadlineNanos - System.nanoTime();
+ if (remaining <= 0)
+ throw new IllegalStateException("Timed out waiting for pending backend proxy deliveries");
+ try {
+ java.util.concurrent.TimeUnit.NANOSECONDS.timedWait(this, remaining);
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Interrupted while waiting for pending backend proxy deliveries", interrupted);
+ }
+ }
+ }
+
+ /** Holds staged replacement sends until its predecessor FIFO can be prepended. */
+ public synchronized void beginPreparedTransportHandoff() {
+ if (transport instanceof HttpBackendProxyTransport http) {
+ http.beginPreparedHandoff();
+ } else {
+ preparedSendFence = true;
+ }
+ }
+
+ /** Admits the predecessor before staged replacement messages without caller-thread I/O. */
+ private void acceptPreparedHandoffMessages(java.util.List pending) {
+ if (transport instanceof HttpBackendProxyTransport http) {
+ http.acceptHandoffMessages(pending);
+ return;
+ }
+ Thread worker = null;
+ boolean schedulePluginMessages = false;
+ long generation = 0;
+ synchronized (this) {
+ if (!preparedSendFence)
+ throw new IllegalStateException("Replacement transport is not awaiting a prepared handoff");
+ int admitted = pending.size() + preparedSends.size();
+ if (admitted > MAX_ASYNC_HANDOFF_SENDS - asyncHandoffSends.size())
+ throw new IllegalStateException("Backend proxy handoff queue exceeded its fixed capacity");
+ asyncHandoffSends.addAll(pending);
+ asyncHandoffSends.addAll(preparedSends);
+ preparedSends.clear();
+ preparedSendFence = false;
+ if (!asyncHandoffSends.isEmpty() && asyncHandoffWorker == null && !pluginMessageHandoffScheduled) {
+ if (transport instanceof PluginMessagingBackendProxyTransport) {
+ pluginMessageHandoffScheduled = true;
+ schedulePluginMessages = true;
+ generation = handoffGeneration;
+ } else {
+ worker = createAsyncHandoffWorker();
+ }
+ }
+ }
+ if (schedulePluginMessages) schedulePluginMessageHandoff(generation);
+ else startAsyncHandoffWorker(worker);
+ }
+
+ /** Preserves handoff FIFO and keeps plugin-message API access on the primary thread. */
+ private void acceptQueuedTransportSend(JsonEnvelope envelope) {
+ if (asyncHandoffSends.size() >= MAX_ASYNC_HANDOFF_SENDS) {
+ if (!preparedQueueWarning) {
+ preparedQueueWarning = true;
+ plugin.getLogger().severe("Plugin-message delivery queue is full; delivery was not accepted");
+ }
+ return;
+ }
+ asyncHandoffSends.addLast(envelope);
+ if (transport instanceof PluginMessagingBackendProxyTransport) {
+ if (pluginMessageHandoffScheduled) return;
+ pluginMessageHandoffScheduled = true;
+ long generation = handoffGeneration;
+ try {
+ schedulePluginMessageHandoff(generation);
+ } catch (RuntimeException failure) {
+ pluginMessageHandoffScheduled = false;
+ notifyAll();
+ throw failure;
+ }
+ } else if (asyncHandoffWorker == null) {
+ startAsyncHandoffWorker(createAsyncHandoffWorker());
+ }
+ }
+
+ private Thread createAsyncHandoffWorker() {
+ Thread worker = new Thread(this::drainAsyncHandoffMessages,
+ "VotingPlugin-Backend-Transport-Handoff");
+ worker.setDaemon(true);
+ asyncHandoffWorker = worker;
+ return worker;
+ }
+
+ private void startAsyncHandoffWorker(Thread worker) {
+ if (worker == null) return;
+ try {
+ worker.start();
+ } catch (RuntimeException failure) {
+ synchronized (this) {
+ if (asyncHandoffWorker == worker) asyncHandoffWorker = null;
+ notifyAll();
+ }
+ throw failure;
+ }
+ }
+
+ private void drainAsyncHandoffMessages() {
+ try {
+ while (!Thread.currentThread().isInterrupted()) {
+ JsonEnvelope envelope;
+ BackendProxyTransport target;
+ synchronized (this) {
+ envelope = asyncHandoffSends.peekFirst();
+ if (envelope == null) {
+ if (asyncHandoffWorker == Thread.currentThread()) asyncHandoffWorker = null;
+ notifyAll();
+ return;
+ }
+ target = transport;
+ }
+ if (target == null) return;
+ boolean accepted;
+ try {
+ accepted = target.send(envelope);
+ } catch (RuntimeException sendFailure) {
+ accepted = false;
+ }
+ synchronized (this) {
+ if (!accepted) {
+ if (!asyncHandoffRetryWarning && plugin != null && plugin.getLogger() != null) {
+ asyncHandoffRetryWarning = true;
+ plugin.getLogger().warning(
+ "Backend proxy handoff delivery was rejected; retaining it for retry");
+ }
+ try {
+ wait(250L);
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ return;
+ }
+ continue;
+ }
+ asyncHandoffRetryWarning = false;
+ if (asyncHandoffSends.peekFirst() == envelope) asyncHandoffSends.removeFirst();
+ notifyAll();
+ }
+ }
+ } finally {
+ synchronized (this) {
+ if (asyncHandoffWorker == Thread.currentThread()) asyncHandoffWorker = null;
+ notifyAll();
+ }
+ }
+ }
+
+ private void schedulePluginMessageHandoff(long generation) {
+ try {
+ plugin.getBukkitScheduler().runTask(plugin, () -> drainPluginMessageHandoff(generation));
+ } catch (RuntimeException failure) {
+ synchronized (this) {
+ if (handoffGeneration == generation) pluginMessageHandoffScheduled = false;
+ notifyAll();
+ }
+ throw failure;
+ }
+ }
+
+ private void drainPluginMessageHandoff(long generation) {
+ for (int sent = 0; sent < PLUGIN_MESSAGE_HANDOFF_BATCH_SIZE; sent++) {
+ JsonEnvelope envelope;
+ BackendProxyTransport target;
+ synchronized (this) {
+ if (handoffGeneration != generation) return;
+ envelope = asyncHandoffSends.peekFirst();
+ if (envelope == null) {
+ pluginMessageHandoffScheduled = false;
+ preparedQueueWarning = false;
+ notifyAll();
+ return;
+ }
+ target = transport;
+ }
+ if (!(target instanceof PluginMessagingBackendProxyTransport)) {
+ synchronized (this) {
+ if (handoffGeneration == generation) pluginMessageHandoffScheduled = false;
+ notifyAll();
+ }
+ return;
+ }
+ try {
+ if (!target.send(envelope)) {
+ schedulePluginMessageHandoff(generation);
+ return;
+ }
+ } catch (RuntimeException failure) {
+ synchronized (this) {
+ if (handoffGeneration == generation) pluginMessageHandoffScheduled = false;
+ notifyAll();
+ }
+ throw failure;
+ }
+ synchronized (this) {
+ if (handoffGeneration != generation) return;
+ if (asyncHandoffSends.peekFirst() == envelope) asyncHandoffSends.removeFirst();
+ notifyAll();
+ }
+ }
+ synchronized (this) {
+ if (handoffGeneration != generation) return;
+ if (asyncHandoffSends.isEmpty()) {
+ pluginMessageHandoffScheduled = false;
+ preparedQueueWarning = false;
+ notifyAll();
+ return;
+ }
+ }
+ schedulePluginMessageHandoff(generation);
+ }
+
+ /** Reserves replacement capacity for every old queued message and future prepared send. */
+ public synchronized void reservePreparedTransportHandoff(BackendProxyTransportManager replacement) {
+ if (preparedTransport == null && !preparedSendFence) return;
+ if (!(java.util.Objects.requireNonNull(replacement, "replacement").transport
+ instanceof HttpBackendProxyTransport target))
+ throw new IllegalStateException("HTTP replacement transport is unavailable");
+ // send() remains available while the previous credential is fenced. Reserve
+ // its whole remaining bounded allowance, not only the current queue size.
+ int previousMessages = preparedTransport instanceof HttpBackendProxyTransport previous
+ ? previous.preparedMessageCount() : 0;
+ target.reservePreparedHandoffCapacity(previousMessages + MAX_PREPARED_SENDS);
+ }
+
+ public void beginPreparedHttpHandoff() {
+ beginPreparedTransportHandoff();
+ }
+
+ public synchronized void restorePreparedTransport() {
+ if (preparedTransport == null) {
+ if (!preparedSendFence) return;
+ acceptPreparedHandoffMessages(java.util.Collections.emptyList());
+ preparedQueueWarning = false;
+ return;
+ }
+ if (transport != null) return;
+ if (preparedTransport instanceof HttpBackendProxyTransport http) {
+ transport = http.recreatePrepared();
+ } else if (preparedTransport instanceof SocketBackendProxyTransport socket) {
+ socket.restoreAfterFailedReplacement();
+ transport = socket;
+ } else if (preparedTransport instanceof MqttBackendProxyTransport mqtt) {
+ mqtt.restoreAfterFailedReplacement();
+ transport = mqtt;
+ } else {
+ throw new IllegalStateException("Prepared backend proxy transport cannot be restored");
+ }
+ preparedTransport = null;
+ preparedSendFence = false;
+ while (!preparedSends.isEmpty() && transport.send(preparedSends.peekFirst())) preparedSends.removeFirst();
+ preparedQueueWarning = false;
+ }
+
+ public void restoreAfterFailedReplacement() {
+ restoreAfterFailedReplacement(null);
+ }
+
+ /** Restores an old Redis listener together with a promoted replacement's unplayed replay FIFO. */
+ public void restoreAfterFailedReplacement(BackendProxyTransportManager failedReplacement) {
+ java.util.List replacementReplay = failedReplacement == null
+ ? java.util.Collections.emptyList() : failedReplacement.detachRedisReplayForFailedHandoff();
+ restoreRetiredRedisAfterFailedHandoff(replacementReplay);
+ restoreCompletedRedisAfterFailedHandoff(replacementReplay);
+ if (transport instanceof PluginMessagingBackendProxyTransport pluginMessaging) {
+ pluginMessaging.restoreAfterFailedReplacement();
+ }
+ restorePreparedTransport();
+ }
+
+ private synchronized java.util.List detachRedisReplayForFailedHandoff() {
+ if (!(transport instanceof RedisBackendProxyTransport redis)) return java.util.Collections.emptyList();
+ return redis.detachReplayForFailedHandoff();
+ }
+
+ public void awaitPreparedTransportRestoration(long deadlineNanos) {
+ if (transport instanceof HttpBackendProxyTransport http) http.awaitCredentialRestoration(deadlineNanos);
+ }
+
+ /** Requires an off-thread drain before abandoning active Redis replay for another transport. */
+ public synchronized boolean hasPendingRedisReplay() {
+ return transport instanceof RedisBackendProxyTransport redis && redis.hasPendingReplayForReplacement();
+ }
+
+ public boolean prepareRedisReplayTransition(BungeeMethod replacementMethod, long deadlineNanos) {
+ RedisBackendProxyTransport redis;
+ synchronized (this) {
+ if (replacementMethod == BungeeMethod.REDIS || !(transport instanceof RedisBackendProxyTransport)) return true;
+ redis = (RedisBackendProxyTransport) transport;
+ }
+ if (!redis.awaitReplayDrainForNonRedisReplacement(deadlineNanos)) return false;
+ synchronized (this) {
+ if (transport != redis) return false;
+ preparedSendFence = true;
+ }
+ return true;
+ }
+
+ public java.util.List closeRedisForHandoff(BackendProxyTransportManager replacement) {
+ RedisBackendProxyTransport retiring;
+ synchronized (this) {
+ if (!(transport instanceof RedisBackendProxyTransport)) {
+ throw new IllegalStateException("Redis backend proxy transport is unavailable");
+ }
+ retiring = (RedisBackendProxyTransport) transport;
+ retiredTransport = retiring;
+ // The Redis listener is fenced off-thread before Bukkit publishes the
+ // staged replacement. Keep sends accepted in that interval in the same
+ // bounded predecessor FIFO instead of dropping them once transport is
+ // detached below. Publication transfers this queue ahead of messages the
+ // staged replacement accepted, while rollback drains it through the
+ // restored Redis transport.
+ preparedSendFence = true;
+ }
+ java.util.List replay = retiring.freezeReplayForSuccessiveHandoff();
+ try {
+ replacement.acceptRedisReplayFromPreviousHandoff(replay);
+ } catch (RuntimeException admissionFailure) {
+ retiring.restoreFrozenReplayAfterFailedSuccessiveHandoff(replay);
+ synchronized (this) {
+ if (retiredTransport == retiring) retiredTransport = null;
+ }
+ throw admissionFailure;
+ }
+ try {
+ // closeForHandoff() waits for already-running Redis callbacks. Those callbacks
+ // may publish a reply through send(), so never retain this manager's monitor
+ // while waiting for them to drain.
+ retiring.closeForHandoff();
+ } catch (RuntimeException failure) {
+ synchronized (this) {
+ if (failure instanceof RedisBackendProxyTransport.HandoffQuiescenceException
+ && retiredTransport == retiring && transport == retiring) {
+ replacement.removeRedisReplayFromPreviousHandoff(replay);
+ retiring.restoreFrozenReplayAfterFailedSuccessiveHandoff(replay);
+ retiredTransport = null;
+ } else if (transport == retiring) {
+ transport = null;
+ }
+ }
+ if (failure instanceof RedisBackendProxyTransport.HandoffQuiescenceException) throw failure;
+ // Retain the fenced old listener so a later manager close can retry its
+ // cleanup without ever touching the promoted replacement.
+ if (plugin != null) {
+ plugin.getLogger().warning("Previous Redis backend listener did not stop cleanly after handoff");
+ plugin.debug(failure);
+ }
+ }
+ synchronized (this) {
+ if (transport == retiring) transport = null;
+ }
+ return replay;
+ }
+
+ private void acceptRedisReplayFromPreviousHandoff(java.util.List replay) {
+ if (transport instanceof RedisBackendProxyTransport redis) redis.acceptReplayFromPreviousHandoff(replay);
+ else if (!replay.isEmpty()) throw new IllegalStateException("Redis replacement transport is unavailable");
+ }
+
+ private void removeRedisReplayFromPreviousHandoff(java.util.List replay) {
+ if (transport instanceof RedisBackendProxyTransport redis) redis.removeReplayFromPreviousHandoff(replay);
}
public void activateRedisAfterHandoff() {
@@ -93,6 +667,90 @@ public void activateRedisAfterHandoff() {
((RedisBackendProxyTransport) transport).activateAfterHandoff();
}
+ public void replayRedisAfterHandoffPublication() {
+ if (transport instanceof RedisBackendProxyTransport redis) redis.replayAfterHandoffPublication();
+ }
+
+ /** Fences the old Redis listener before promoting the validated standby. */
+ public void completeRedisHandoff(BackendProxyTransportManager replacement) {
+ java.util.Objects.requireNonNull(replacement, "replacement");
+ java.util.List replay = closeRedisForHandoff(replacement);
+ try {
+ replacement.activateRedisAfterHandoff();
+ } catch (RuntimeException activationFailure) {
+ java.util.List rollbackReplay = replacement.detachRedisReplayForFailedHandoff();
+ restoreRetiredRedisAfterFailedHandoff(activationFailure,
+ mergeRedisRollbackReplay(replay, rollbackReplay));
+ throw activationFailure;
+ }
+ closeRetiredRedisAfterHandoff();
+ }
+
+ private static java.util.List mergeRedisRollbackReplay(java.util.List predecessorReplay,
+ java.util.List standbyReplay) {
+ if (standbyReplay.isEmpty()) return predecessorReplay;
+ if (predecessorReplay.isEmpty()) return standbyReplay;
+ if (standbyReplay.size() >= predecessorReplay.size()
+ && standbyReplay.subList(0, predecessorReplay.size()).equals(predecessorReplay)) return standbyReplay;
+ java.util.ArrayList merged = new java.util.ArrayList<>(
+ predecessorReplay.size() + standbyReplay.size());
+ merged.addAll(predecessorReplay);
+ merged.addAll(standbyReplay);
+ if (merged.size() > RedisBackendProxyTransport.MAX_REPLAY_HANDOFF_DELIVERIES)
+ throw new IllegalStateException("Redis rollback replay exceeds its bounded handoff capacity");
+ return merged;
+ }
+
+ private synchronized void restoreRetiredRedisAfterFailedHandoff(RuntimeException activationFailure,
+ java.util.List replacementReplay) {
+ try {
+ restoreRetiredRedisAfterFailedHandoff(replacementReplay);
+ } catch (RuntimeException restorationFailure) {
+ activationFailure.addSuppressed(restorationFailure);
+ }
+ }
+
+ private synchronized void restoreRetiredRedisAfterFailedHandoff(java.util.List replacementReplay) {
+ if (transport != null) return;
+ if (!(retiredTransport instanceof RedisBackendProxyTransport redis)) return;
+ try {
+ redis.restoreAfterFailedHandoff(replacementReplay);
+ transport = redis;
+ retiredTransport = null;
+ } catch (RuntimeException restorationFailure) {
+ throw new IllegalStateException("Retired Redis backend listener could not be restored", restorationFailure);
+ }
+ }
+
+ private synchronized void closeRetiredRedisAfterHandoff() {
+ if (retiredTransport == null) return;
+ BackendProxyTransport retired = retiredTransport;
+ retiredTransport = null;
+ try {
+ retired.close();
+ if (retired instanceof RedisBackendProxyTransport redis) completedRedisHandoffTransport = redis;
+ } catch (RuntimeException cleanupFailure) {
+ retiredTransport = retired;
+ if (plugin != null) {
+ plugin.getLogger().warning("Retired Redis backend listener did not stop cleanly after handoff");
+ plugin.debug(cleanupFailure);
+ }
+ }
+ }
+
+ /** Restores a worker-retired Redis listener when publication did not commit. */
+ private synchronized void restoreCompletedRedisAfterFailedHandoff(java.util.List replacementReplay) {
+ if (transport != null || completedRedisHandoffTransport == null) return;
+ RedisBackendProxyTransport retired = completedRedisHandoffTransport;
+ try {
+ retired.restoreAfterFailedHandoff(replacementReplay);
+ transport = retired;
+ completedRedisHandoffTransport = null;
+ } catch (RuntimeException restorationFailure) {
+ throw new IllegalStateException("Retired Redis backend listener could not be restored", restorationFailure);
+ }
+ }
+
public ClientHandler getClientHandler() {
return transport instanceof SocketBackendProxyTransport
? ((SocketBackendProxyTransport) transport).getClientHandler() : null;
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java
new file mode 100644
index 000000000..919c96f54
--- /dev/null
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java
@@ -0,0 +1,783 @@
+package com.bencodez.votingplugin.backendproxy.transport;
+
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.time.Clock;
+import java.util.ArrayDeque;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope;
+import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler;
+import com.bencodez.simpleapi.servercomm.http.HttpBackendTransportConnector;
+import com.bencodez.simpleapi.servercomm.http.HttpClientCredentialStore;
+import com.bencodez.simpleapi.servercomm.http.HttpConnectionCode;
+import com.bencodez.simpleapi.servercomm.http.HttpTlsIdentity;
+import com.bencodez.votingplugin.VotingPluginMain;
+import com.bencodez.votingplugin.util.DurableFiles;
+
+/** Backend adapter for the secure outbound-only HTTP proxy transport. */
+public final class HttpBackendProxyTransport implements BackendProxyTransport {
+ private static final int MAX_STARTUP_QUEUE = 1024;
+ private static final int MAX_PREPUBLICATION_QUEUE = 2048;
+ private static final int MAX_HANDOFF_QUEUE = 4096;
+ private static final long DEFAULT_STARTUP_VALIDATION_SECONDS = 25L;
+ private static final long ENROLLMENT_RETRY_INITIAL_MILLIS = 1_000L;
+ private static final long ENROLLMENT_RETRY_MAX_MILLIS = 60_000L;
+ private static final long INCOMING_DISPATCH_SECONDS = 25L;
+ private static final long SHUTDOWN_FLUSH_SECONDS = 5L;
+ private static final ConcurrentHashMap DIRECTORY_OWNERS = new ConcurrentHashMap<>();
+ private final VotingPluginMain plugin;
+ private final Object lifecycle = new Object();
+ private final CountDownLatch startupComplete = new CountDownLatch(1);
+ private final CountDownLatch credentialRestoreComplete = new CountDownLatch(1);
+ private final ArrayDeque startupQueue = new ArrayDeque<>();
+ private final ArrayDeque handoffQueue = new ArrayDeque<>();
+ private volatile Thread handoffWorker;
+ private boolean awaitingPreparedHandoff;
+ /** Capacity reserved for the prepared predecessor before this transport is published. */
+ private int preparedHandoffReservation;
+ private volatile HttpBackendTransportConnector connector;
+ private volatile Thread worker;
+ private volatile RuntimeException startupFailure;
+ private volatile RuntimeException credentialRestoreFailure;
+ private volatile boolean started;
+ private volatile boolean closed;
+ /** True only after this transport has crossed the handler publication boundary. */
+ private boolean published;
+ private volatile boolean restartAfterFailedFlush;
+ private final java.util.concurrent.atomic.AtomicBoolean flushRecoveryRunning = new java.util.concurrent.atomic.AtomicBoolean();
+ private Path configuredDirectory;
+ private String configuredServerId;
+ private String configuredConnectionCode;
+ private GlobalMessageHandler configuredMessageHandler;
+ private HttpClientCredentialStore.ActiveCredentialGeneration configuredCredentialGeneration;
+ private HttpClientCredentialStore.ActiveCredentialGeneration credentialGenerationToRestore;
+ private boolean retryInitialization;
+ private boolean restoreUnenrolledState;
+ private boolean inboundActive;
+ private Semaphore directoryOwner;
+ private final java.util.concurrent.atomic.AtomicBoolean queueWarning = new java.util.concurrent.atomic.AtomicBoolean();
+
+ public HttpBackendProxyTransport(VotingPluginMain plugin) {
+ this.plugin = plugin;
+ }
+
+ @Override
+ public void start(GlobalMessageHandler messageHandler) {
+ start(messageHandler, true);
+ }
+
+ @Override
+ public void start(GlobalMessageHandler messageHandler, boolean retryInitialization) {
+ Path directory = plugin.getDataFolder().toPath().resolve("http");
+ String serverId = plugin.getBungeeSettings().getServer();
+ String connectionCode = plugin.getBungeeSettings().getHttpConnectionCode();
+ start(directory, serverId, connectionCode, messageHandler, null, retryInitialization, false,
+ retryInitialization);
+ }
+
+ private void start(Path directory, String serverId, String connectionCode,
+ GlobalMessageHandler messageHandler) {
+ start(directory, serverId, connectionCode, messageHandler, null, true, false, true);
+ }
+
+ private void start(Path directory, String serverId, String connectionCode,
+ GlobalMessageHandler messageHandler,
+ HttpClientCredentialStore.ActiveCredentialGeneration generationToRestore) {
+ start(directory, serverId, connectionCode, messageHandler, generationToRestore, false, false, true);
+ }
+
+ private void start(Path directory, String serverId, String connectionCode,
+ GlobalMessageHandler messageHandler,
+ HttpClientCredentialStore.ActiveCredentialGeneration generationToRestore,
+ boolean retryInitialization, boolean restoreUnenrolledState, boolean inboundActive) {
+ if (generationToRestore == null && !restoreUnenrolledState)
+ validateConfiguration(directory, serverId, connectionCode);
+ else HttpTlsIdentity.canonicalServerId(serverId);
+ configuredDirectory = directory;
+ configuredServerId = serverId;
+ configuredConnectionCode = connectionCode;
+ configuredMessageHandler = messageHandler;
+ credentialGenerationToRestore = generationToRestore;
+ this.retryInitialization = retryInitialization;
+ this.restoreUnenrolledState = restoreUnenrolledState;
+ this.inboundActive = inboundActive;
+ this.published = inboundActive;
+ started = true;
+ worker = new Thread(() -> initialize(directory, serverId, connectionCode, messageHandler, retryInitialization,
+ restoreUnenrolledState),
+ "VotingPlugin-HTTP-Backend-Setup");
+ worker.setDaemon(true);
+ worker.start();
+ }
+
+ HttpBackendProxyTransport recreatePrepared() {
+ HttpBackendProxyTransport restored = new HttpBackendProxyTransport(plugin);
+ synchronized (lifecycle) {
+ // Startup and handoff queues are one FIFO from the caller's perspective.
+ // The handoff queue can still contain messages accepted by the previous
+ // replacement, so rollback must carry it into the restored transport too.
+ // Keep the combined queue in the asynchronously drained handoff lane: it can
+ // be larger than the connector's bounded startup queue.
+ restored.handoffQueue.addAll(takeQueuedMessages());
+ }
+ restored.start(configuredDirectory, configuredServerId, configuredConnectionCode, configuredMessageHandler,
+ configuredCredentialGeneration, configuredCredentialGeneration == null && retryInitialization,
+ restoreUnenrolledState, true);
+ return restored;
+ }
+
+ boolean isClosedForReplacement() {
+ return closed;
+ }
+
+ java.util.List drainPreparedMessages() {
+ synchronized (lifecycle) {
+ return takeQueuedMessages();
+ }
+ }
+
+ int preparedMessageCount() {
+ synchronized (lifecycle) {
+ return startupQueue.size() + handoffQueue.size();
+ }
+ }
+
+ java.util.List preparedMessagesSnapshot() {
+ synchronized (lifecycle) {
+ java.util.List pending = new java.util.ArrayList<>(startupQueue.size() + handoffQueue.size());
+ pending.addAll(startupQueue);
+ pending.addAll(handoffQueue);
+ return pending;
+ }
+ }
+
+ /** Takes both pending queues in their original FIFO order for replacement handoff. */
+ private java.util.List takeQueuedMessages() {
+ java.util.List pending = new java.util.ArrayList<>(startupQueue.size() + handoffQueue.size());
+ pending.addAll(startupQueue);
+ pending.addAll(handoffQueue);
+ startupQueue.clear();
+ handoffQueue.clear();
+ return pending;
+ }
+
+ @Override
+ public void prepareForReplacement() {
+ HttpBackendTransportConnector active;
+ Thread cancelledInitialization = null;
+ /*
+ * Keep the check, credential snapshot, and cancellation in one lifecycle
+ * critical section. During ordinary first-time enrollment the setup worker
+ * owns DIRECTORY_OWNERS while it retries and connector is intentionally null.
+ * A Control replacement must be able to cancel that worker before claiming
+ * the same credential directory; there is no credential to preserve in that
+ * state. If enrollment has already published a credential but the connector
+ * has not yet been installed, retain the generation just as we do for an
+ * established connector.
+ */
+ synchronized (lifecycle) {
+ if (configuredDirectory == null)
+ throw new IllegalStateException("Could not preserve the active HTTP client credential before it became ready");
+ active = connector;
+ if (active == null && !HttpClientCredentialStore.hasEnrolledProfile(configuredDirectory)) {
+ restoreUnenrolledState = true;
+ cancelledInitialization = worker;
+ closeForReplacement();
+ } else try {
+ configuredCredentialGeneration = HttpClientCredentialStore.snapshotActiveGeneration(configuredDirectory);
+ } catch (Exception failure) {
+ throw new IllegalStateException("Could not preserve the active HTTP client credential", failure);
+ }
+ if (cancelledInitialization == null && active == null) {
+ closeForReplacement();
+ return;
+ }
+ }
+ if (cancelledInitialization != null) {
+ awaitCancelledInitialization(cancelledInitialization);
+ captureEnrollmentPublishedDuringCancellation();
+ return;
+ }
+ flushForReplacement(active, System.nanoTime() + TimeUnit.SECONDS.toNanos(SHUTDOWN_FLUSH_SECONDS));
+ closeForReplacement();
+ }
+
+ private void awaitCancelledInitialization(Thread setup) {
+ try { setup.join(TimeUnit.SECONDS.toMillis(SHUTDOWN_FLUSH_SECONDS)); }
+ catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException("Interrupted while stopping HTTP client enrollment", interrupted);
+ }
+ if (setup.isAlive())
+ throw new IllegalStateException("Could not stop HTTP client enrollment before replacement");
+ }
+
+ private void captureEnrollmentPublishedDuringCancellation() {
+ if (!HttpClientCredentialStore.hasEnrolledProfile(configuredDirectory)) return;
+ try {
+ HttpConnectionCode original = HttpConnectionCode.parse(configuredConnectionCode);
+ if (!original.serverId().equals(HttpTlsIdentity.canonicalServerId(configuredServerId))
+ || !HttpClientCredentialStore.matchesEnrollmentCode(configuredDirectory, original))
+ throw new IllegalStateException("HTTP client enrollment changed during replacement preparation");
+ configuredCredentialGeneration = HttpClientCredentialStore.snapshotActiveGeneration(configuredDirectory);
+ restoreUnenrolledState = false;
+ } catch (IllegalStateException failure) { throw failure; }
+ catch (Exception failure) {
+ throw new IllegalStateException("Could not preserve the completed HTTP client enrollment", failure);
+ }
+ }
+
+ void flushForReplacement(HttpBackendTransportConnector connector, long deadlineNanos) {
+ if (connector.flushOutgoing(deadlineNanos)) return;
+ restartAfterFailedFlush = true;
+ connector.start();
+ if (flushRecoveryRunning.compareAndSet(false, true)) {
+ Thread recovery = new Thread(() -> resumeAfterFailedFlush(connector),
+ "VotingPlugin-HTTP-Backend-Flush-Recovery");
+ recovery.setDaemon(true);
+ recovery.start();
+ }
+ throw new IllegalStateException("Could not drain the active HTTP transport before replacement");
+ }
+
+ private void resumeAfterFailedFlush(HttpBackendTransportConnector active) {
+ try {
+ // start() intentionally does nothing while the interrupted long-poll worker is
+ // still winding down. Keep a single recovery owner alive until this transport is
+ // closed or replaced so the connector's already-accepted queue cannot be stranded.
+ while (!closed && connector == active) {
+ active.start();
+ synchronized (lifecycle) {
+ while (!startupQueue.isEmpty() && active.send(startupQueue.peekFirst())) startupQueue.removeFirst();
+ }
+ try { TimeUnit.SECONDS.sleep(1); }
+ catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); break; }
+ }
+ } finally {
+ flushRecoveryRunning.set(false);
+ if (closed || connector != active) restartAfterFailedFlush = false;
+ }
+ }
+
+ private void initialize(Path directory, String serverId, String configuredCode,
+ GlobalMessageHandler messageHandler, boolean retryEnrollment, boolean restoreUnenrolledState) {
+ Path ownerKey = directory.toAbsolutePath().normalize();
+ Semaphore owner = DIRECTORY_OWNERS.computeIfAbsent(ownerKey, ignored -> new Semaphore(1));
+ boolean acquired = false, installed = false;
+ HttpBackendTransportConnector replacement = null;
+ try {
+ owner.acquire();
+ acquired = true;
+ synchronized (lifecycle) {
+ if (closed) {
+ if (credentialGenerationToRestore != null || restoreUnenrolledState)
+ credentialRestoreFailure = new IllegalStateException(
+ "Previous HTTP client credential state restoration was cancelled");
+ return;
+ }
+ }
+ try {
+ if (restoreUnenrolledState)
+ restoreUnenrolledCredentialState(directory, serverId, configuredCode);
+ if (credentialGenerationToRestore != null) {
+ HttpClientCredentialStore.restoreActiveGenerationAfterReplacement(directory,
+ credentialGenerationToRestore);
+ }
+ } catch (Exception failure) {
+ credentialRestoreFailure = new IllegalStateException(
+ "Could not restore the previous HTTP client credential state", failure);
+ throw failure;
+ }
+ credentialRestoreComplete.countDown();
+ // An enrolled rollback resumes the validated generation without replaying its
+ // temporary code. An un-enrolled rollback reuses the original startup code
+ // only after the staged credential has been made inactive.
+ HttpConnectionCode code = enrollmentCode(directory, serverId,
+ credentialGenerationToRestore == null ? configuredCode : null);
+ if (code != null && !enrollForStartup(code, serverId, directory, retryEnrollment,
+ this::waitForEnrollmentRetry)) return;
+ HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(directory);
+ if (!enrolled.profile().serverId().equals(HttpTlsIdentity.canonicalServerId(serverId)))
+ throw new IllegalStateException("Persisted HTTP identity belongs to a different backend Server name");
+ replacement = new HttpBackendTransportConnector(directory, envelope -> {
+ dispatchAfterPublication(messageHandler, envelope);
+ });
+ invokeConnectorLifecycle(replacement, "startPaused");
+ boolean discard = false;
+ synchronized (lifecycle) {
+ if (closed) {
+ discard = true;
+ } else {
+ transferStartupQueue(startupQueue, replacement);
+ connector = replacement;
+ directoryOwner = owner;
+ installed = true;
+ if (inboundActive) invokeConnectorLifecycle(replacement, "activateIncoming");
+ }
+ }
+ if (!discard) startHandoffDrainIfNeeded();
+ } catch (Exception failure) {
+ startupFailure = new IllegalStateException("Secure HTTP backend enrollment or connection failed", failure);
+ plugin.getLogger().severe("Secure HTTP backend transport is unavailable; check the connection code and proxy endpoint");
+ } finally {
+ credentialRestoreComplete.countDown();
+ finishInitialization(installed, replacement, acquired, owner, startupComplete);
+ }
+ }
+
+ static void finishInitialization(boolean installed, HttpBackendTransportConnector replacement,
+ boolean acquired, Semaphore owner, CountDownLatch completion) {
+ try {
+ if (!installed) {
+ try {
+ if (replacement != null) replacement.close();
+ } finally {
+ if (acquired) owner.release();
+ }
+ }
+ } finally {
+ completion.countDown();
+ }
+ }
+
+ static void transferStartupQueue(ArrayDeque queue, HttpBackendTransportConnector connector) {
+ while (!queue.isEmpty()) {
+ JsonEnvelope envelope = queue.peekFirst();
+ if (!connector.send(envelope)) {
+ throw new IllegalStateException("HTTP startup queue could not be transferred");
+ }
+ queue.removeFirst();
+ }
+ }
+
+ @Override
+ public void activateAfterPublication() {
+ HttpBackendTransportConnector active;
+ synchronized (lifecycle) {
+ if (closed) return;
+ active = connector;
+ }
+ if (active != null) invokeConnectorLifecycle(active, "activateIncoming");
+ synchronized (lifecycle) {
+ if (closed) return;
+ inboundActive = true;
+ published = true;
+ lifecycle.notifyAll();
+ }
+ }
+
+ private boolean awaitInboundPublication() {
+ synchronized (lifecycle) {
+ while (!inboundActive && !closed) {
+ try {
+ lifecycle.wait();
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ return false;
+ }
+ }
+ return !closed;
+ }
+ }
+
+ void dispatchAfterPublication(GlobalMessageHandler messageHandler, JsonEnvelope envelope) {
+ if (!awaitInboundPublication())
+ throw new IllegalStateException("HTTP transport closed before inbound publication");
+ dispatchIncoming(messageHandler, envelope,
+ System.nanoTime() + TimeUnit.SECONDS.toNanos(INCOMING_DISPATCH_SECONDS));
+ }
+
+ private static void invokeConnectorLifecycle(HttpBackendTransportConnector connector, String method) {
+ try {
+ connector.getClass().getMethod(method).invoke(connector);
+ } catch (java.lang.reflect.InvocationTargetException failure) {
+ Throwable cause = failure.getCause();
+ if (cause instanceof RuntimeException runtime) throw runtime;
+ if (cause instanceof Error error) throw error;
+ throw new IllegalStateException("HTTP connector " + method + " failed", cause);
+ } catch (NoSuchMethodException unavailable) {
+ // Older published SimpleAPI snapshots do not yet expose the publication
+ // barrier. The wrapper callback above provides the same fence until #79 is
+ // deployed, while newer versions use the native connector barrier.
+ if ("startPaused".equals(method)) connector.start();
+ else if (!"activateIncoming".equals(method))
+ throw new IllegalStateException("SimpleAPI HTTP connector does not support " + method, unavailable);
+ } catch (ReflectiveOperationException failure) {
+ throw new IllegalStateException("SimpleAPI HTTP connector does not support " + method, failure);
+ }
+ }
+
+ static void restoreUnenrolledCredentialState(Path directory, String serverId, String configuredCode) throws Exception {
+ Path root = directory.toAbsolutePath().normalize();
+ if (configuredCode != null && !configuredCode.isBlank()
+ && HttpClientCredentialStore.hasEnrolledProfile(root)) {
+ try {
+ HttpConnectionCode original = HttpConnectionCode.parse(configuredCode);
+ if (original.serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))
+ && HttpClientCredentialStore.matchesEnrollmentCode(root, original)) return;
+ } catch (Exception ignored) {
+ // The pre-replacement state was un-enrolled; never retain a credential
+ // that cannot be tied to its already-validated original code.
+ }
+ }
+ Path current = root.resolve("http-transport-client-current").normalize();
+ if (!current.getParent().equals(root) || Files.isSymbolicLink(current))
+ throw new java.io.IOException("HTTP client credential pointer is unsafe");
+ if (Files.exists(current, LinkOption.NOFOLLOW_LINKS)
+ && !Files.isRegularFile(current, LinkOption.NOFOLLOW_LINKS))
+ throw new java.io.IOException("HTTP client credential pointer is unsafe");
+ DurableFiles.deleteIfExists(current);
+ if (HttpClientCredentialStore.hasEnrolledProfile(root))
+ throw new java.io.IOException("HTTP client credential rollback did not restore the un-enrolled state");
+ }
+
+ private boolean waitForEnrollmentRetry(long delayMillis) {
+ if (closed) return false;
+ try {
+ TimeUnit.MILLISECONDS.sleep(delayMillis);
+ return !closed;
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ return false;
+ }
+ }
+
+ /** Performs initial enrollment with a bounded backoff for ordinary startup. */
+ boolean enrollForStartup(HttpConnectionCode code, String serverId, Path directory,
+ boolean retryEnrollment, java.util.function.LongPredicate waitForRetry) throws Exception {
+ long retryDelayMillis = ENROLLMENT_RETRY_INITIAL_MILLIS;
+ while (true) {
+ if (closed) return false;
+ try {
+ HttpBackendTransportConnector.enroll(code, serverId, directory);
+ return true;
+ } catch (Exception failure) {
+ if (!retryEnrollment) throw failure;
+ plugin.getLogger().warning("Secure HTTP backend enrollment failed; retrying with bounded backoff");
+ if (!waitForRetry.test(retryDelayMillis)) return false;
+ retryDelayMillis = Math.min(ENROLLMENT_RETRY_MAX_MILLIS, retryDelayMillis * 2L);
+ }
+ }
+ }
+
+ void dispatchIncoming(GlobalMessageHandler messageHandler, JsonEnvelope envelope, long deadlineNanos) {
+ CountDownLatch completed = new CountDownLatch(1);
+ AtomicReference failure = new AtomicReference<>();
+ AtomicInteger state = new AtomicInteger(0); // pending, running, cancelled, finished
+ try {
+ plugin.getBukkitScheduler().runTask(plugin, () -> {
+ if (!state.compareAndSet(0, 1)) {
+ completed.countDown();
+ return;
+ }
+ try {
+ messageHandler.onMessage(envelope);
+ } catch (Throwable thrown) {
+ failure.set(thrown);
+ } finally {
+ state.set(3);
+ completed.countDown();
+ }
+ });
+ } catch (Throwable rejected) {
+ throw new IllegalStateException("Could not schedule an incoming HTTP message on the server thread", rejected);
+ }
+ try {
+ long remaining = deadlineNanos - System.nanoTime();
+ if (remaining <= 0L || !completed.await(remaining, TimeUnit.NANOSECONDS)) {
+ state.compareAndSet(0, 2);
+ throw new IllegalStateException("Incoming HTTP message handling exceeded its delivery deadline");
+ }
+ } catch (InterruptedException interrupted) {
+ state.compareAndSet(0, 2);
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException("Incoming HTTP message handling was interrupted", interrupted);
+ }
+ Throwable thrown = failure.get();
+ if (thrown != null)
+ throw new IllegalStateException("Incoming HTTP message handling failed", thrown);
+ }
+
+ void awaitCredentialRestoration(long deadlineNanos) {
+ if (credentialGenerationToRestore == null && !restoreUnenrolledState) return;
+ try {
+ long remaining = deadlineNanos - System.nanoTime();
+ if (remaining <= 0L || !credentialRestoreComplete.await(remaining, TimeUnit.NANOSECONDS))
+ throw new IllegalStateException("Previous HTTP client credential restoration timed out");
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException("Previous HTTP client credential restoration was interrupted", interrupted);
+ }
+ RuntimeException failure = credentialRestoreFailure;
+ if (failure != null) throw failure;
+ }
+
+ @Override
+ public boolean send(JsonEnvelope envelope) {
+ synchronized (lifecycle) {
+ if (closed) return false;
+ if (awaitingPreparedHandoff || !handoffQueue.isEmpty()) {
+ int capacity = awaitingPreparedHandoff
+ ? Math.min(MAX_PREPUBLICATION_QUEUE, MAX_HANDOFF_QUEUE - preparedHandoffReservation)
+ : MAX_HANDOFF_QUEUE;
+ if (handoffQueue.size() < capacity) {
+ handoffQueue.addLast(envelope);
+ return true;
+ }
+ warnRejectedSend();
+ return false;
+ }
+ HttpBackendTransportConnector active = connector;
+ if (active != null) {
+ if (!active.send(envelope)) {
+ if (restartAfterFailedFlush && startupQueue.size() < MAX_STARTUP_QUEUE) {
+ startupQueue.addLast(envelope);
+ return true;
+ }
+ warnRejectedSend();
+ return false;
+ }
+ } else if (startupQueue.size() < MAX_STARTUP_QUEUE) {
+ startupQueue.addLast(envelope);
+ return true;
+ } else {
+ warnRejectedSend();
+ return false;
+ }
+ return true;
+ }
+ }
+
+ void beginPreparedHandoff() {
+ synchronized (lifecycle) {
+ if (closed) throw new IllegalStateException("HTTP replacement transport is closed");
+ awaitingPreparedHandoff = true;
+ }
+ }
+
+ /**
+ * Reserves enough of the bounded handoff queue for the predecessor before callers
+ * can send through this staged replacement. This turns an otherwise late,
+ * destructive capacity failure into a pre-publication validation failure.
+ */
+ void reservePreparedHandoffCapacity(int messages) {
+ if (messages < 0 || messages > MAX_HANDOFF_QUEUE)
+ throw new IllegalStateException("HTTP prepared handoff exceeds its fixed capacity");
+ synchronized (lifecycle) {
+ if (closed) throw new IllegalStateException("HTTP replacement transport is closed");
+ if (!awaitingPreparedHandoff)
+ throw new IllegalStateException("HTTP replacement transport is not awaiting a prepared handoff");
+ if (handoffQueue.size() > MAX_HANDOFF_QUEUE - messages)
+ throw new IllegalStateException("HTTP handoff queue exceeded its reserved capacity");
+ preparedHandoffReservation = messages;
+ }
+ }
+
+ void acceptHandoffMessages(java.util.List messages) {
+ synchronized (lifecycle) {
+ if (closed) throw new IllegalStateException("HTTP replacement transport is closed");
+ if (messages.size() > MAX_HANDOFF_QUEUE - handoffQueue.size())
+ throw new IllegalStateException("HTTP handoff queue exceeded its fixed capacity");
+ java.util.ArrayDeque newer = new java.util.ArrayDeque<>(handoffQueue);
+ handoffQueue.clear();
+ handoffQueue.addAll(messages);
+ handoffQueue.addAll(newer);
+ awaitingPreparedHandoff = false;
+ preparedHandoffReservation = 0;
+ }
+ startHandoffDrainIfNeeded();
+ }
+
+ private void startHandoffDrainIfNeeded() {
+ Thread drain;
+ synchronized (lifecycle) {
+ if (closed || handoffQueue.isEmpty() || handoffWorker != null || connector == null) return;
+ drain = new Thread(this::drainHandoffMessages, "VotingPlugin-HTTP-Backend-Handoff");
+ drain.setDaemon(true);
+ handoffWorker = drain;
+ }
+ drain.start();
+ }
+
+ private void drainHandoffMessages() {
+ try {
+ while (!Thread.currentThread().isInterrupted()) {
+ boolean delivered = false;
+ synchronized (lifecycle) {
+ if (closed || handoffQueue.isEmpty()) return;
+ HttpBackendTransportConnector active = connector;
+ if (active != null && active.send(handoffQueue.peekFirst())) {
+ handoffQueue.removeFirst();
+ delivered = true;
+ }
+ }
+ if (!delivered) TimeUnit.MILLISECONDS.sleep(25L);
+ }
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ } finally {
+ synchronized (lifecycle) {
+ if (handoffWorker == Thread.currentThread()) handoffWorker = null;
+ }
+ }
+ }
+
+ java.util.List handoffMessagesSnapshot() {
+ synchronized (lifecycle) {
+ return java.util.List.copyOf(handoffQueue);
+ }
+ }
+
+ private void warnRejectedSend() {
+ if (queueWarning.compareAndSet(false, true))
+ plugin.getLogger().severe("Secure HTTP transport queue is full or rejected an oversized message; delivery was not accepted");
+ }
+
+ @Override
+ public void validate() {
+ validate(System.nanoTime() + TimeUnit.SECONDS.toNanos(DEFAULT_STARTUP_VALIDATION_SECONDS));
+ }
+
+ void validate(long deadlineNanos) {
+ String serverId = plugin.getBungeeSettings().getServer();
+ if (serverId == null || !serverId.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) {
+ throw new IllegalStateException("HTTP requires a valid unique backend Server name");
+ }
+ Path directory = plugin.getDataFolder().toPath().resolve("http");
+ validateConfiguration(directory, serverId, plugin.getBungeeSettings().getHttpConnectionCode());
+ if (!started) return;
+ try {
+ long remaining = deadlineNanos - System.nanoTime();
+ if (remaining <= 0L || !startupComplete.await(remaining, TimeUnit.NANOSECONDS))
+ throw new IllegalStateException("Secure HTTP backend setup did not finish within the validation deadline");
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException("Secure HTTP backend setup validation was interrupted", interrupted);
+ }
+ RuntimeException failure = startupFailure;
+ if (failure != null) throw failure;
+ HttpBackendTransportConnector active = connector;
+ if (closed || active == null) throw new IllegalStateException("Secure HTTP backend transport did not initialize");
+ try {
+ if (!active.awaitFirstResponse(deadlineNanos))
+ throw new IllegalStateException("HTTP backend could not authenticate with the proxy");
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException("Secure HTTP backend readiness validation was interrupted", interrupted);
+ }
+ }
+
+ public static void validateConfiguration(Path directory, String serverId, String configuredCode) {
+ enrollmentCode(directory, serverId, configuredCode);
+ }
+
+ static HttpConnectionCode enrollmentCode(Path directory, String serverId, String configuredCode) {
+ try { serverId = HttpTlsIdentity.canonicalServerId(serverId); }
+ catch (IllegalArgumentException invalid) { throw new IllegalStateException("HTTP requires a valid unique backend Server name", invalid); }
+ boolean enrolled = HttpClientCredentialStore.hasEnrolledProfile(directory);
+ if (configuredCode != null && !configuredCode.isBlank()) {
+ try {
+ HttpConnectionCode code = HttpConnectionCode.parse(configuredCode);
+ if (!code.serverId().equals(serverId))
+ throw new IllegalArgumentException("Connection code belongs to a different backend");
+ if (enrolled && HttpClientCredentialStore.matchesEnrollmentCode(directory, code)) return null;
+ code.requireActive(Clock.systemUTC());
+ return code;
+ } catch (Exception invalid) {
+ throw new IllegalStateException("HTTP ConnectionCode is invalid, expired, or belongs to a different backend", invalid);
+ }
+ }
+ if (!enrolled)
+ throw new IllegalStateException("HTTP requires a temporary ConnectionCode for initial enrollment");
+ return null;
+ }
+
+ @Override
+ public void close() {
+ close(true);
+ }
+
+ private void closeForReplacement() {
+ close(false);
+ }
+
+ private void close(boolean discardQueuedMessages) {
+ Thread setup;
+ Thread pendingHandoff;
+ HttpBackendTransportConnector active;
+ Semaphore owner;
+ java.util.List finalHandoff;
+ synchronized (lifecycle) {
+ if (closed) return;
+ closed = true;
+ lifecycle.notifyAll();
+ if (discardQueuedMessages) {
+ if (published) {
+ finalHandoff = new java.util.ArrayList<>(startupQueue.size() + handoffQueue.size());
+ finalHandoff.addAll(startupQueue);
+ finalHandoff.addAll(handoffQueue);
+ } else {
+ // A staged replacement never became authoritative. Its presence and
+ // handoff messages must not be flushed after rollback.
+ finalHandoff = java.util.List.of();
+ }
+ startupQueue.clear();
+ handoffQueue.clear();
+ } else finalHandoff = java.util.List.of();
+ awaitingPreparedHandoff = false;
+ setup = worker;
+ worker = null;
+ pendingHandoff = handoffWorker;
+ handoffWorker = null;
+ active = connector;
+ connector = null;
+ owner = directoryOwner;
+ directoryOwner = null;
+ }
+ startupComplete.countDown();
+ if (setup != null) setup.interrupt();
+ if (pendingHandoff != null) pendingHandoff.interrupt();
+ if (setup == null && active == null && owner == null) return;
+ Thread cleanup = new Thread(() -> drain(setup, active, owner, finalHandoff),
+ "VotingPlugin-HTTP-Backend-Cleanup");
+ cleanup.setDaemon(true);
+ cleanup.start();
+ }
+
+ private static void drain(Thread setup, HttpBackendTransportConnector active, Semaphore owner,
+ java.util.List finalHandoff) {
+ try {
+ if (setup != null) try { setup.join(TimeUnit.SECONDS.toMillis(5)); }
+ catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); }
+ if (active != null) {
+ flushHandoffForShutdown(active, finalHandoff,
+ System.nanoTime() + TimeUnit.SECONDS.toNanos(SHUTDOWN_FLUSH_SECONDS));
+ active.close();
+ }
+ } finally { if (owner != null) owner.release(); }
+ }
+
+ static boolean flushHandoffForShutdown(HttpBackendTransportConnector active,
+ java.util.List finalHandoff, long deadlineNanos) {
+ for (JsonEnvelope envelope : finalHandoff) {
+ while (!active.send(envelope)) {
+ long remaining = deadlineNanos - System.nanoTime();
+ if (remaining <= 0L) return false;
+ try {
+ TimeUnit.NANOSECONDS.sleep(Math.min(remaining, TimeUnit.MILLISECONDS.toNanos(25L)));
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ return false;
+ }
+ }
+ }
+ return active.flushOutgoing(deadlineNanos);
+ }
+}
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MqttBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MqttBackendProxyTransport.java
index 5b9bb44a0..85e2d1253 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MqttBackendProxyTransport.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MqttBackendProxyTransport.java
@@ -15,6 +15,13 @@ public class MqttBackendProxyTransport implements BackendProxyTransport {
private final VotingPluginMain plugin;
@Getter
private MqttHandler mqttHandler;
+ private GlobalMessageHandler messageHandler;
+ private String publishTopic;
+ private String subscriptionTopic;
+ private String clientId;
+ private String brokerUrl;
+ private String username;
+ private String password;
public MqttBackendProxyTransport(VotingPluginMain plugin) {
this.plugin = plugin;
@@ -23,14 +30,18 @@ public MqttBackendProxyTransport(VotingPluginMain plugin) {
@Override
public void start(GlobalMessageHandler messageHandler) {
try {
- String id = plugin.getBungeeSettings().getMqttClientID();
- if (id.isEmpty()) {
- id = plugin.getOptions().getServer();
+ this.messageHandler = messageHandler;
+ publishTopic = plugin.getBungeeSettings().getMqttPrefix() + "votingplugin/servers/proxy";
+ subscriptionTopic = plugin.getBungeeSettings().getMqttPrefix() + "votingplugin/servers/"
+ + plugin.getOptions().getServer();
+ clientId = plugin.getBungeeSettings().getMqttClientID();
+ if (clientId.isEmpty()) {
+ clientId = plugin.getOptions().getServer();
}
- mqttHandler = new MqttHandler(new MqttServerComm(id, plugin.getBungeeSettings().getMqttBrokerURL(),
- plugin.getBungeeSettings().getMqttUsername(), plugin.getBungeeSettings().getMqttPassword()), 2);
- mqttHandler.subscribeEnvelopes(plugin.getBungeeSettings().getMqttPrefix() + "votingplugin/servers/"
- + plugin.getOptions().getServer(), (topic, envelope) -> messageHandler.onMessage(envelope));
+ brokerUrl = plugin.getBungeeSettings().getMqttBrokerURL();
+ username = plugin.getBungeeSettings().getMqttUsername();
+ password = plugin.getBungeeSettings().getMqttPassword();
+ startCapturedConnection();
} catch (MqttException e) {
throw new IllegalStateException("MQTT backend proxy transport initialization failed", e);
} catch (Exception e) {
@@ -38,21 +49,53 @@ public void start(GlobalMessageHandler messageHandler) {
}
}
+ protected MqttHandler createMqttHandler(MqttServerComm server) throws MqttException {
+ return new MqttHandler(server, 2);
+ }
+
+ protected MqttServerComm createMqttServerComm() throws MqttException {
+ return new MqttServerComm(clientId, brokerUrl, username, password);
+ }
+
+ private void startCapturedConnection() throws Exception {
+ MqttHandler candidate = createMqttHandler(createMqttServerComm());
+ try {
+ candidate.subscribeEnvelopes(subscriptionTopic, (topic, envelope) -> messageHandler.onMessage(envelope));
+ mqttHandler = candidate;
+ } catch (Exception subscriptionFailure) {
+ try {
+ candidate.disconnect();
+ } catch (Exception disconnectFailure) {
+ subscriptionFailure.addSuppressed(disconnectFailure);
+ }
+ throw subscriptionFailure;
+ }
+ }
+
@Override
public void validate() {
if (mqttHandler == null) throw new IllegalStateException("MQTT backend proxy transport initialization failed");
}
+ /** Returns whether a failed disconnect left the existing broker session usable. */
+ public boolean isConnected() {
+ return mqttHandler != null && mqttHandler.isConnected();
+ }
+
@Override
- public void send(JsonEnvelope envelope) {
+ public boolean send(JsonEnvelope envelope) {
if (mqttHandler == null) {
- return;
+ return false;
}
try {
- mqttHandler.publishEnvelope(plugin.getBungeeSettings().getMqttPrefix() + "votingplugin/servers/proxy",
- envelope);
+ mqttHandler.publishEnvelope(publishTopic, envelope);
+ return true;
} catch (Exception e) {
- e.printStackTrace();
+ if (plugin != null && plugin.getLogger() != null) {
+ plugin.getLogger().warning("MQTT backend proxy delivery failed");
+ plugin.debug(e);
+ }
+ return false;
}
}
@@ -80,4 +123,16 @@ public void close() {
}
}
}
+
+ /** Reconnects a prepared predecessor with its original identity after rollback. */
+ public void restoreAfterFailedReplacement() {
+ if (messageHandler == null || clientId == null || brokerUrl == null || subscriptionTopic == null) {
+ throw new IllegalStateException("MQTT backend proxy transport cannot be restored before startup");
+ }
+ try {
+ startCapturedConnection();
+ } catch (Exception e) {
+ throw new IllegalStateException("MQTT backend proxy transport restoration failed", e);
+ }
+ }
}
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MysqlBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MysqlBackendProxyTransport.java
index dc61e9f70..0cbf34be2 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MysqlBackendProxyTransport.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MysqlBackendProxyTransport.java
@@ -43,14 +43,19 @@ public void validate() {
}
@Override
- public void send(JsonEnvelope envelope) {
+ public boolean send(JsonEnvelope envelope) {
if (messenger == null) {
- return;
+ return false;
}
try {
messenger.sendToProxy(envelope);
+ return true;
} catch (SQLException e) {
- e.printStackTrace();
+ if (plugin != null && plugin.getLogger() != null) {
+ plugin.getLogger().warning("MySQL backend proxy delivery failed");
+ plugin.debug(e);
+ }
+ return false;
}
}
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java
index d6a24ff5c..5979e0baf 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java
@@ -11,6 +11,10 @@ public class PluginMessagingBackendProxyTransport implements BackendProxyTranspo
private final VotingPluginMain plugin;
private GlobalMessageHandler messageHandler;
+ private String channel;
+ private EncryptionHandler encryptionHandler;
+ private boolean debug;
+ private boolean active;
public PluginMessagingBackendProxyTransport(VotingPluginMain plugin) {
this.plugin = plugin;
@@ -19,27 +23,53 @@ public PluginMessagingBackendProxyTransport(VotingPluginMain plugin) {
@Override
public void start(GlobalMessageHandler messageHandler) {
this.messageHandler = messageHandler;
- plugin.registerBungeeChannels(plugin.getBungeeSettings().getPluginMessagingChannel());
- EncryptionHandler encryptionHandler = null;
+ channel = plugin.getBungeeSettings().getPluginMessagingChannel();
+ encryptionHandler = null;
if (plugin.getBungeeSettings().isPluginMessageEncryption()) {
encryptionHandler = new EncryptionHandler(plugin.getName(),
new File(plugin.getDataFolder(), "secretkey.key"));
}
+ debug = plugin.getBungeeSettings().isBungeeDebug();
+ }
+
+ @Override
+ public void activateAfterPublication() {
+ if (messageHandler != null && !active) {
+ publishSharedState();
+ active = true;
+ }
+ }
+
+ void restoreAfterFailedReplacement() {
+ if (messageHandler != null) {
+ publishSharedState();
+ active = true;
+ }
+ }
+
+ private void publishSharedState() {
+ if (plugin.getPluginMessaging() == null || !channel.equals(plugin.getBungeeChannel())) {
+ plugin.registerBungeeChannels(channel);
+ }
plugin.getPluginMessaging().setEncryptionHandler(encryptionHandler);
- plugin.getPluginMessaging().setDebug(plugin.getBungeeSettings().isBungeeDebug());
+ plugin.getPluginMessaging().setDebug(debug);
plugin.activateBackendPluginMessageHandler(messageHandler);
}
@Override
- public void send(JsonEnvelope envelope) {
+ public boolean send(JsonEnvelope envelope) {
plugin.getPluginMessaging().sendEnvelope(envelope);
+ return true;
}
@Override
public void close() {
- if (messageHandler != null) {
+ if (messageHandler != null && active) {
plugin.deactivateBackendPluginMessageHandler(messageHandler);
- messageHandler = null;
}
+ active = false;
+ messageHandler = null;
+ channel = null;
+ encryptionHandler = null;
}
}
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java
index eb2ce2539..2e9de9272 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java
@@ -22,7 +22,29 @@
import lombok.Getter;
public class RedisBackendProxyTransport implements BackendProxyTransport {
+ public static final class HandoffQuiescenceException extends IllegalStateException {
+ private static final long serialVersionUID = 1L;
+
+ public HandoffQuiescenceException(String message) {
+ super(message);
+ }
+ }
+
+ public static final class HandoffReplayBackpressureException extends IllegalStateException {
+ private static final long serialVersionUID = 1L;
+
+ public HandoffReplayBackpressureException(String message) {
+ super(message);
+ }
+ }
static final int MAX_LEGACY_HANDOFF_DELIVERIES = 4096;
+ static final int MAX_IDENTIFIED_HANDOFF_DELIVERIES = 4096;
+ static final int MAX_REPLAY_HANDOFF_DELIVERIES = MAX_LEGACY_HANDOFF_DELIVERIES
+ + MAX_IDENTIFIED_HANDOFF_DELIVERIES;
+ static final int REPLAY_BATCH_SIZE = 32;
+ private static final long HANDOFF_QUIESCE_TIMEOUT_NANOS = TimeUnit.SECONDS.toNanos(3);
+ private static final long HANDOFF_REPLAY_BACKPRESSURE_TIMEOUT_NANOS = TimeUnit.SECONDS.toNanos(3);
+ private static final long REPLAY_BATCH_EXECUTION_TIMEOUT_NANOS = TimeUnit.SECONDS.toNanos(3);
private final VotingPluginMain plugin;
private final ProcessedVoteCache processedVoteCache;
@@ -32,10 +54,33 @@ public class RedisBackendProxyTransport implements BackendProxyTransport {
private Thread listenerThread;
private final Object subscriberIdentity = new Object();
private final Object legacyLifecycle = new Object();
- private final List bufferedLegacyDeliveries = new ArrayList<>();
+ private final List bufferedLegacyDeliveries = new ArrayList<>();
+ private final List bufferedIdentifiedDeliveries = new ArrayList<>();
private long bufferedLegacyDeliveryBytes;
+ private long bufferedIdentifiedDeliveryBytes;
+ private long nextHandoffSequence;
+ private boolean standbySubscriber;
+ private boolean identifiedHandoffOverflowed;
+ private boolean legacyHandoffOverflowed;
private boolean legacyHandoffDegraded;
+ private boolean retiredAfterHandoff;
+ private boolean replayingHandoff;
+ private boolean replayTransferFrozen;
+ private boolean replayBackpressureFailureLogged;
+ private long replayGeneration;
+ private Thread replayWorker;
+ private boolean replayTaskOutstanding;
+ private long replayTaskGeneration = -1L;
+ private int replayDeliveriesInFlight;
+ // Counts only handlers which have already crossed the Bukkit dispatch fence.
+ // It is deliberately independent of replayGeneration so cancellation cannot
+ // erase accounting for a callback that was already executing.
+ private int replayCallbacksInFlight;
+ private int dispatchesInFlight;
+ private final java.util.ArrayDeque deliveriesAfterReplay = new java.util.ArrayDeque<>();
private GlobalMessageHandler messageHandler;
+ private GlobalMessageHandler handoffMessageHandler;
+ private String publishChannel;
public RedisBackendProxyTransport(VotingPluginMain plugin) {
this(plugin, new ProcessedVoteCache());
@@ -49,7 +94,9 @@ public RedisBackendProxyTransport(VotingPluginMain plugin, ProcessedVoteCache pr
@Override
public void start(GlobalMessageHandler messageHandler) {
this.messageHandler = messageHandler;
- processedVoteCache.registerRedisSubscriber(subscriberIdentity);
+ publishChannel = plugin.getBungeeSettings().getRedisPrefix() + "VotingPlugin";
+ retiredAfterHandoff = false;
+ standbySubscriber = !processedVoteCache.registerRedisSubscriber(subscriberIdentity);
redisHandler = new RedisHandler(plugin.getBungeeSettings().getRedisHost(),
plugin.getBungeeSettings().getRedisPort(), plugin.getBungeeSettings().getRedisUsername(),
plugin.getBungeeSettings().getRedisPassword(), plugin.getBungeeSettings().getRedisdbindex(),
@@ -70,11 +117,7 @@ public void debug(String message) {
try {
JsonEnvelope envelope = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.decode(payload);
String deliveryId = envelope.getFields().get(VotingPluginWire.K_REDIS_DELIVERY_ID);
- if (deliveryId != null) {
- if (processedVoteCache.reserveRedisDelivery(deliveryId)) messageHandler.onMessage(envelope);
- } else {
- dispatchLegacy(envelope);
- }
+ dispatchReceivedSubscriberEnvelope(envelope, deliveryId);
} catch (Exception e) {
plugin.debug("Redis decode failed: " + e.getMessage());
}
@@ -89,25 +132,127 @@ public void onSubscribe(String channel, int subscribedChannels) {
listenerThread.start();
}
+ /**
+ * Redis Pub/Sub has already consumed this payload when its callback is invoked.
+ * A bounded queue wait therefore cannot discard it: retry the same callback in
+ * place, applying TCP/Pub/Sub backpressure until FIFO replay capacity returns or
+ * this subscriber is deliberately fenced during shutdown/rollback.
+ */
+ void dispatchReceivedSubscriberEnvelope(JsonEnvelope envelope, String deliveryId) {
+ while (true) {
+ try {
+ if (deliveryId != null) dispatchIdentified(envelope, deliveryId);
+ else dispatchLegacy(envelope);
+ return;
+ } catch (HandoffReplayBackpressureException retry) {
+ synchronized (legacyLifecycle) {
+ if (retiredAfterHandoff) return;
+ }
+ if (plugin != null)
+ plugin.debug("Redis handoff replay remains full; retaining the received Pub/Sub payload for retry");
+ }
+ }
+ }
+
+ void dispatchIdentified(JsonEnvelope envelope, String deliveryId) {
+ boolean accepted;
+ synchronized (legacyLifecycle) {
+ if (!awaitReplayTransferResolution()) return;
+ if (retiredAfterHandoff) return;
+ awaitReplayCapacity();
+ if (retiredAfterHandoff) return;
+ if (standbySubscriber) {
+ bufferIdentifiedDelivery(envelope, deliveryId);
+ return;
+ }
+ accepted = processedVoteCache.reserveRedisDelivery(deliveryId);
+ if (accepted && replayingHandoff) {
+ enqueueReplayDelivery(envelope);
+ accepted = false;
+ }
+ if (accepted) dispatchesInFlight++;
+ }
+ if (accepted) dispatchTracked(envelope);
+ }
+
void dispatchLegacy(JsonEnvelope envelope) {
- String signature = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.encode(envelope);
+ // A mixed-version publisher may put the reliable-only identity on a payload
+ // that is delivered through the legacy path. Do not expose that identity to
+ // handlers or let it alter legacy duplicate accounting.
+ JsonEnvelope legacyEnvelope = withoutReliableFields(envelope);
+ String signature = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.encode(legacyEnvelope);
int encodedBytes = ProcessedVoteCache.legacyRedisDeliveryBytes(signature);
+ boolean dispatch = false;
synchronized (legacyLifecycle) {
+ if (!awaitReplayTransferResolution()) return;
+ if (retiredAfterHandoff) return;
+ awaitReplayCapacity();
+ if (retiredAfterHandoff) return;
if (processedVoteCache.reserveLegacyRedisDelivery(subscriberIdentity, signature)) {
- messageHandler.onMessage(envelope);
+ dispatch = true;
} else if (encodedBytes <= ProcessedVoteCache.MAX_LEGACY_REDIS_DELIVERY_BYTES
&& bufferedLegacyDeliveries.size() < MAX_LEGACY_HANDOFF_DELIVERIES
+ && handoffDeliveryCount() < MAX_REPLAY_HANDOFF_DELIVERIES
&& bufferedLegacyDeliveryBytes <= ProcessedVoteCache.MAX_LEGACY_REDIS_TOTAL_BYTES - encodedBytes) {
- bufferedLegacyDeliveries.add(envelope);
+ bufferedLegacyDeliveries.add(new BufferedHandoffDelivery(
+ nextHandoffSequence++, legacyEnvelope, signature, false));
bufferedLegacyDeliveryBytes += encodedBytes;
} else {
- if (!legacyHandoffDegraded && plugin != null) {
- plugin.getLogger().warning("Redis legacy handoff exceeded its " + MAX_LEGACY_HANDOFF_DELIVERIES
- + " delivery / " + ProcessedVoteCache.MAX_LEGACY_REDIS_TOTAL_BYTES
- + " byte buffer; temporarily degrading duplicate suppression");
+ if (standbySubscriber) {
+ if (!legacyHandoffOverflowed && plugin != null)
+ plugin.getLogger().warning("Redis legacy handoff buffer is full; aborting the staged handoff");
+ legacyHandoffOverflowed = true;
+ } else {
+ if (!legacyHandoffDegraded && plugin != null) {
+ plugin.getLogger().warning("Redis legacy handoff exceeded its " + MAX_LEGACY_HANDOFF_DELIVERIES
+ + " delivery / " + ProcessedVoteCache.MAX_LEGACY_REDIS_TOTAL_BYTES
+ + " byte buffer; temporarily degrading duplicate suppression");
+ }
+ legacyHandoffDegraded = true;
+ dispatch = true;
}
- legacyHandoffDegraded = true;
- messageHandler.onMessage(envelope);
+ }
+ if (dispatch && replayingHandoff) {
+ enqueueReplayDelivery(envelope);
+ dispatch = false;
+ }
+ if (dispatch) dispatchesInFlight++;
+ }
+ if (dispatch) dispatchTracked(legacyEnvelope);
+ }
+
+ private static JsonEnvelope withoutReliableFields(JsonEnvelope envelope) {
+ if (!envelope.getFields().containsKey(VotingPluginWire.K_REDIS_DELIVERY_ID)) return envelope;
+ JsonEnvelope.Builder builder = JsonEnvelope.builder(envelope.getSubChannel()).schema(envelope.getSchema());
+ for (java.util.Map.Entry field : envelope.getFields().entrySet()) {
+ if (!VotingPluginWire.K_REDIS_DELIVERY_ID.equals(field.getKey())) builder.put(field.getKey(), field.getValue());
+ }
+ return builder.build();
+ }
+
+ /** Holds a consumed subscriber callback until a same-Redis transfer commits or rolls back. */
+ private boolean awaitReplayTransferResolution() {
+ boolean interrupted = false;
+ while (replayTransferFrozen) {
+ try {
+ legacyLifecycle.wait();
+ } catch (InterruptedException waitInterrupted) {
+ // Do not drop a Pub/Sub payload merely because the listener was nudged
+ // during transfer. Its final owner will explicitly release this wait.
+ interrupted = true;
+ }
+ }
+ if (interrupted) Thread.currentThread().interrupt();
+ return !retiredAfterHandoff;
+ }
+
+ private void dispatchTracked(JsonEnvelope envelope) {
+ try {
+ messageHandler.onMessage(envelope);
+ } finally {
+ synchronized (legacyLifecycle) {
+ dispatchesInFlight--;
+ legacyLifecycle.notifyAll();
}
}
}
@@ -115,29 +260,596 @@ void dispatchLegacy(JsonEnvelope envelope) {
/** Promotes a validated standby after the previous listener has completely stopped. */
public void activateAfterHandoff() {
synchronized (legacyLifecycle) {
+ if (identifiedHandoffOverflowed || legacyHandoffOverflowed
+ || processedVoteCache.isLegacyRedisHandoffOverflowed())
+ throw new IllegalStateException("Redis handoff buffer overflowed before publication");
+ replayingHandoff = true;
+ replayBackpressureFailureLogged = false;
processedVoteCache.activateRedisSubscriber(subscriberIdentity);
- for (JsonEnvelope envelope : bufferedLegacyDeliveries) {
- String signature = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.encode(envelope);
- if (!processedVoteCache.consumeLegacyRedisDelivery(signature)) messageHandler.onMessage(envelope);
+ standbySubscriber = false;
+ java.util.ArrayList buffered = new java.util.ArrayList<>(
+ bufferedIdentifiedDeliveries.size() + bufferedLegacyDeliveries.size());
+ buffered.addAll(bufferedIdentifiedDeliveries);
+ buffered.addAll(bufferedLegacyDeliveries);
+ buffered.sort(java.util.Comparator.comparingLong(BufferedHandoffDelivery::sequence));
+ for (BufferedHandoffDelivery delivery : buffered) {
+ boolean dispatch;
+ try {
+ dispatch = delivery.identified()
+ ? processedVoteCache.reserveRedisDelivery(delivery.identity())
+ : !processedVoteCache.consumeLegacyRedisDelivery(delivery.identity());
+ } catch (RuntimeException replayFailure) {
+ if (plugin != null) plugin.debug("Redis handoff replay failed: " + replayFailure.getMessage());
+ continue;
+ }
+ if (dispatch) enqueueReplayDelivery(delivery.envelope());
}
+ bufferedIdentifiedDeliveries.clear();
+ bufferedIdentifiedDeliveryBytes = 0;
bufferedLegacyDeliveries.clear();
bufferedLegacyDeliveryBytes = 0;
+ identifiedHandoffOverflowed = false;
+ legacyHandoffOverflowed = false;
legacyHandoffDegraded = false;
processedVoteCache.finishRedisHandoff();
}
}
+ /** Replays buffered deliveries after the owning handler opens its publication gate. */
+ public void replayAfterHandoffPublication() {
+ if (plugin == null) {
+ // Unit tests use a transport without a Bukkit scheduler. Production always
+ // takes the bounded worker path below.
+ drainReplayWithoutScheduler();
+ return;
+ }
+ startReplayWorkerIfNeeded();
+ }
+
+ /** True while a same-Redis replay still owns accepted envelopes or Bukkit work. */
+ public boolean hasPendingReplayForReplacement() {
+ synchronized (legacyLifecycle) {
+ return replayingHandoff || replayTransferFrozen || !deliveriesAfterReplay.isEmpty()
+ || replayDeliveriesInFlight != 0 || replayCallbacksInFlight != 0 || replayTaskOutstanding;
+ }
+ }
+
+ /**
+ * Waits only on the Control/validation worker for active replay to finish
+ * before a non-Redis replacement can retire this transport. It never moves
+ * Redis envelopes into a transport that cannot preserve Redis semantics.
+ */
+ public boolean awaitReplayDrainForNonRedisReplacement(long deadlineNanos) {
+ boolean interrupted = false;
+ synchronized (legacyLifecycle) {
+ while (hasPendingReplayForReplacement()) {
+ long remaining = deadlineNanos - System.nanoTime();
+ if (remaining <= 0L) {
+ if (interrupted) Thread.currentThread().interrupt();
+ return false;
+ }
+ try {
+ TimeUnit.NANOSECONDS.timedWait(legacyLifecycle, remaining);
+ } catch (InterruptedException waitInterrupted) {
+ interrupted = true;
+ break;
+ }
+ }
+ }
+ if (interrupted) Thread.currentThread().interrupt();
+ return !interrupted;
+ }
+
+ private void startReplayWorkerIfNeeded() {
+ Thread worker;
+ synchronized (legacyLifecycle) {
+ if (!replayingHandoff || replayTransferFrozen || replayWorker != null || replayTaskOutstanding) return;
+ worker = new Thread(this::drainReplayOnWorker, "VotingPlugin-Redis-Backend-Replay");
+ worker.setDaemon(true);
+ replayWorker = worker;
+ }
+ try {
+ worker.start();
+ } catch (RuntimeException failed) {
+ synchronized (legacyLifecycle) {
+ if (replayWorker == worker) replayWorker = null;
+ legacyLifecycle.notifyAll();
+ }
+ throw failed;
+ }
+ }
+
+ private void enqueueReplayDelivery(JsonEnvelope envelope) {
+ if (deliveriesAfterReplay.size() + replayDeliveriesInFlight >= MAX_REPLAY_HANDOFF_DELIVERIES)
+ throw new IllegalStateException("Redis handoff replay capacity was not reserved");
+ deliveriesAfterReplay.addLast(envelope);
+ }
+
+ /**
+ * Preserves handoff FIFO without allowing the replay queue to grow without
+ * bound. A timeout is explicit and happens before the identified ID is
+ * reserved, so an undeliverable callback is never silently marked processed.
+ */
+ private void awaitReplayCapacity() {
+ if (!replayingHandoff) return;
+ boolean interrupted = false;
+ long deadline = System.nanoTime() + HANDOFF_REPLAY_BACKPRESSURE_TIMEOUT_NANOS;
+ while (replayingHandoff && deliveriesAfterReplay.size() + replayDeliveriesInFlight
+ >= MAX_REPLAY_HANDOFF_DELIVERIES) {
+ long remaining = deadline - System.nanoTime();
+ if (remaining <= 0L) {
+ if (!replayBackpressureFailureLogged && plugin != null) {
+ plugin.getLogger().severe("Redis handoff replay is not draining; rejected an unreserved callback");
+ replayBackpressureFailureLogged = true;
+ }
+ if (interrupted) Thread.currentThread().interrupt();
+ throw new HandoffReplayBackpressureException("Redis handoff replay queue did not drain before its deadline");
+ }
+ try {
+ TimeUnit.NANOSECONDS.timedWait(legacyLifecycle, remaining);
+ } catch (InterruptedException waitInterrupted) {
+ interrupted = true;
+ break;
+ }
+ }
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ throw new HandoffReplayBackpressureException("Interrupted while waiting for Redis handoff replay capacity");
+ }
+ }
+
+ private void drainReplayWithoutScheduler() {
+ while (true) {
+ java.util.ArrayList replay;
+ GlobalMessageHandler handler;
+ long generation;
+ synchronized (legacyLifecycle) {
+ if (!replayingHandoff || deliveriesAfterReplay.isEmpty()) {
+ replayingHandoff = false;
+ legacyLifecycle.notifyAll();
+ return;
+ }
+ replay = takeReplayBatch();
+ handler = messageHandler;
+ generation = replayGeneration;
+ }
+ dispatchReplayBatch(replay, handler, generation);
+ completeReplayBatch(replay.size());
+ }
+ }
+
+ /** Coordinates one bounded Bukkit batch at a time without blocking publication. */
+ private void drainReplayOnWorker() {
+ try {
+ while (!Thread.currentThread().isInterrupted()) {
+ java.util.ArrayList replay;
+ GlobalMessageHandler handler;
+ long generation;
+ synchronized (legacyLifecycle) {
+ if (!replayingHandoff || deliveriesAfterReplay.isEmpty()) {
+ replayingHandoff = false;
+ legacyLifecycle.notifyAll();
+ return;
+ }
+ if (replayTransferFrozen) return;
+ replay = takeReplayBatch();
+ handler = messageHandler;
+ generation = replayGeneration;
+ }
+ ReplayBatch batch = new ReplayBatch(generation);
+ java.util.concurrent.CountDownLatch completed = new java.util.concurrent.CountDownLatch(1);
+ try {
+ synchronized (legacyLifecycle) {
+ replayTaskOutstanding = true;
+ replayTaskGeneration = batch.generation;
+ }
+ plugin.getBukkitScheduler().runTask(plugin, () -> {
+ try {
+ synchronized (legacyLifecycle) {
+ if (batch.generation != replayGeneration || batch.returned || !replayingHandoff) return;
+ if (replayTransferFrozen) return;
+ batch.started = true;
+ }
+ dispatchReplayBatch(replay, handler, batch.generation);
+ } finally {
+ synchronized (legacyLifecycle) {
+ if (batch.generation == replayGeneration && !batch.returned) {
+ if (!batch.started && replayTransferFrozen) {
+ batch.returned = true;
+ requeueReplayBatch(replay);
+ } else completeReplayBatch(replay.size());
+ }
+ if (replayTaskOutstanding && replayTaskGeneration == batch.generation) {
+ replayTaskOutstanding = false;
+ replayTaskGeneration = -1L;
+ }
+ legacyLifecycle.notifyAll();
+ }
+ completed.countDown();
+ startReplayWorkerIfNeeded();
+ }
+ });
+ if (!completed.await(REPLAY_BATCH_EXECUTION_TIMEOUT_NANOS, TimeUnit.NANOSECONDS)) {
+ synchronized (legacyLifecycle) {
+ if (!batch.started && !batch.returned && batch.generation == replayGeneration) {
+ batch.returned = true;
+ replayGeneration++;
+ requeueReplayBatch(replay);
+ // Do not leave replay owned by a Bukkit task that was accepted but
+ // never began. A task already running must retain ownership until its
+ // finally block completes; otherwise a second worker could overlap it
+ // and the older task could clear the newer task's shared state.
+ if (replayTaskOutstanding && replayTaskGeneration == batch.generation) {
+ replayTaskOutstanding = false;
+ replayTaskGeneration = -1L;
+ }
+ }
+ legacyLifecycle.notifyAll();
+ }
+ return;
+ }
+ } catch (InterruptedException interrupted) {
+ synchronized (legacyLifecycle) {
+ // A batch is removed before it can be scheduled. If the worker is
+ // interrupted while waiting for Bukkit, put an unstarted batch back
+ // at the front; otherwise the accepted deliveries would vanish.
+ if (!batch.started && !batch.returned && batch.generation == replayGeneration) {
+ batch.returned = true;
+ requeueReplayBatch(replay);
+ if (replayTaskOutstanding && replayTaskGeneration == batch.generation) {
+ replayTaskOutstanding = false;
+ replayTaskGeneration = -1L;
+ }
+ }
+ legacyLifecycle.notifyAll();
+ }
+ Thread.currentThread().interrupt();
+ return;
+ } catch (RuntimeException schedulingFailure) {
+ synchronized (legacyLifecycle) {
+ if (!batch.returned && batch.generation == replayGeneration) {
+ batch.returned = true;
+ requeueReplayBatch(replay);
+ }
+ if (replayTaskOutstanding && replayTaskGeneration == batch.generation) {
+ replayTaskOutstanding = false;
+ replayTaskGeneration = -1L;
+ }
+ legacyLifecycle.notifyAll();
+ }
+ if (plugin != null) plugin.debug("Redis handoff replay scheduling failed: " + schedulingFailure.getMessage());
+ try {
+ Thread.sleep(250L);
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ return;
+ }
+ }
+ }
+ } finally {
+ synchronized (legacyLifecycle) {
+ if (replayWorker == Thread.currentThread()) replayWorker = null;
+ legacyLifecycle.notifyAll();
+ }
+ startReplayWorkerIfNeeded();
+ }
+ }
+
+ private java.util.ArrayList takeReplayBatch() {
+ java.util.ArrayList replay = new java.util.ArrayList<>(REPLAY_BATCH_SIZE);
+ while (!deliveriesAfterReplay.isEmpty() && replay.size() < REPLAY_BATCH_SIZE)
+ replay.add(deliveriesAfterReplay.removeFirst());
+ replayDeliveriesInFlight += replay.size();
+ return replay;
+ }
+
+ private void completeReplayBatch(int size) {
+ synchronized (legacyLifecycle) {
+ replayDeliveriesInFlight -= size;
+ if (replayDeliveriesInFlight < 0) replayDeliveriesInFlight = 0;
+ legacyLifecycle.notifyAll();
+ }
+ }
+
+ private void requeueReplayBatch(java.util.List replay) {
+ for (int index = replay.size() - 1; index >= 0; index--) deliveriesAfterReplay.addFirst(replay.get(index));
+ completeReplayBatch(replay.size());
+ }
+
+ private static final class ReplayBatch {
+ private final long generation;
+ private boolean started;
+ private boolean returned;
+
+ private ReplayBatch(long generation) {
+ this.generation = generation;
+ }
+ }
+
+ private void dispatchReplayBatch(java.util.List replay, GlobalMessageHandler handler,
+ long generation) {
+ if (handler == null) return;
+ for (JsonEnvelope envelope : replay) {
+ synchronized (legacyLifecycle) {
+ // A close/fence may happen after Bukkit accepts the batch. Never let a
+ // stale generation start another callback after that point.
+ if (!replayingHandoff || generation != replayGeneration) return;
+ replayCallbacksInFlight++;
+ }
+ try {
+ handler.onMessage(envelope);
+ } catch (RuntimeException replayFailure) {
+ if (plugin != null) plugin.debug("Redis handoff replay failed: " + replayFailure.getMessage());
+ } finally {
+ synchronized (legacyLifecycle) {
+ replayCallbacksInFlight--;
+ if (replayCallbacksInFlight < 0) replayCallbacksInFlight = 0;
+ legacyLifecycle.notifyAll();
+ }
+ }
+ }
+ }
+
/** Stops the old listener while retaining its overlap accounting for standby promotion. */
public void closeForHandoff() {
+ handoffMessageHandler = messageHandler;
+ fenceAfterHandoff();
+ // A successful fence guarantees this listener will not dispatch the held
+ // callback. Release it before listener shutdown so the Redis listener thread
+ // can return and join promptly; the overlapping standby owns its duplicate.
+ completeFrozenReplayTransfer();
closeListener(false);
}
+ /** Reopens a fenced active subscriber when standby promotion is rejected. */
+ public void restoreAfterFailedHandoff() {
+ restoreAfterFailedHandoff(java.util.Collections.emptyList());
+ }
+
+ /**
+ * Reinstates the retired subscriber and prepends deliveries accepted by a
+ * promoted replacement before publication later failed. Standby overlap IDs
+ * are resolved while detaching them, so this FIFO can replay directly exactly
+ * once after the old subscriber is restored.
+ */
+ public void restoreAfterFailedHandoff(java.util.List replacementReplay) {
+ GlobalMessageHandler handler = handoffMessageHandler;
+ if (handler == null) throw new IllegalStateException("Redis handoff transport cannot be restored");
+ Thread retiredListener = listenerThread;
+ if (retiredListener != null && retiredListener.isAlive())
+ throw new IllegalStateException("Redis handoff listener did not stop before rollback restart");
+ synchronized (legacyLifecycle) {
+ if (replacementReplay.size() > MAX_REPLAY_HANDOFF_DELIVERIES)
+ throw new IllegalStateException("Redis rollback replay exceeds its bounded handoff capacity");
+ deliveriesAfterReplay.clear();
+ deliveriesAfterReplay.addAll(replacementReplay);
+ replayDeliveriesInFlight = 0;
+ replayingHandoff = !replacementReplay.isEmpty();
+ replayGeneration++;
+ retiredAfterHandoff = false;
+ }
+ processedVoteCache.restoreRedisSubscriber(subscriberIdentity);
+ start(handler);
+ if (!replacementReplay.isEmpty()) replayAfterHandoffPublication();
+ handoffMessageHandler = null;
+ }
+
+ /** Transfers pre-publication replay ownership to a restored predecessor. */
+ public java.util.List detachReplayForFailedHandoff() {
+ synchronized (legacyLifecycle) {
+ if (replayDeliveriesInFlight != 0 || replayCallbacksInFlight != 0)
+ throw new IllegalStateException("Redis rollback cannot detach a replay batch already executing");
+ java.util.ArrayList buffered = new java.util.ArrayList<>(
+ bufferedIdentifiedDeliveries.size() + bufferedLegacyDeliveries.size());
+ buffered.addAll(bufferedIdentifiedDeliveries);
+ buffered.addAll(bufferedLegacyDeliveries);
+ buffered.sort(java.util.Comparator.comparingLong(BufferedHandoffDelivery::sequence));
+ if (deliveriesAfterReplay.size() + buffered.size() > MAX_REPLAY_HANDOFF_DELIVERIES)
+ throw new IllegalStateException("Redis rollback replay exceeds its bounded handoff capacity");
+ java.util.ArrayList pending = new java.util.ArrayList<>(
+ deliveriesAfterReplay.size() + buffered.size());
+ pending.addAll(deliveriesAfterReplay);
+ for (BufferedHandoffDelivery delivery : buffered) {
+ boolean replay;
+ try {
+ replay = delivery.identified()
+ ? processedVoteCache.reserveRedisDelivery(delivery.identity())
+ : !processedVoteCache.consumeLegacyRedisDelivery(delivery.identity());
+ } catch (RuntimeException cacheFailure) {
+ // A rollback must keep an accepted staged callback. Retain it for the
+ // restored subscriber rather than letting a transient dedupe failure
+ // turn replacement close into data loss.
+ replay = true;
+ if (plugin != null) plugin.debug("Redis rollback dedupe failed: " + cacheFailure.getMessage());
+ }
+ if (replay) pending.add(delivery.envelope());
+ }
+ retiredAfterHandoff = true;
+ replayingHandoff = false;
+ replayTransferFrozen = false;
+ replayGeneration++;
+ deliveriesAfterReplay.clear();
+ bufferedIdentifiedDeliveries.clear();
+ bufferedIdentifiedDeliveryBytes = 0;
+ bufferedLegacyDeliveries.clear();
+ bufferedLegacyDeliveryBytes = 0;
+ identifiedHandoffOverflowed = false;
+ legacyHandoffOverflowed = false;
+ legacyHandoffDegraded = false;
+ replayTaskOutstanding = false;
+ replayTaskGeneration = -1L;
+ legacyLifecycle.notifyAll();
+ return pending;
+ }
+ }
+
+ /**
+ * Stops admitting new callbacks and transfers the remaining active replay FIFO
+ * to the next same-Redis standby. An already-started Bukkit batch is allowed
+ * to finish; an unstarted scheduled batch is returned to the deque first.
+ */
+ public java.util.List freezeReplayForSuccessiveHandoff() {
+ boolean interrupted = false;
+ long deadline = System.nanoTime() + HANDOFF_QUIESCE_TIMEOUT_NANOS;
+ synchronized (legacyLifecycle) {
+ replayTransferFrozen = true;
+ retiredAfterHandoff = true;
+ Thread worker = replayWorker;
+ if (worker != null) worker.interrupt();
+ while (replayDeliveriesInFlight != 0 || replayCallbacksInFlight != 0 || replayTaskOutstanding) {
+ long remaining = deadline - System.nanoTime();
+ if (remaining <= 0L) {
+ resumeReplayAfterFailedSuccessiveHandoffLocked();
+ if (interrupted) Thread.currentThread().interrupt();
+ throw new HandoffQuiescenceException("Redis replay did not quiesce before successive handoff");
+ }
+ try {
+ TimeUnit.NANOSECONDS.timedWait(legacyLifecycle, remaining);
+ } catch (InterruptedException waitInterrupted) {
+ interrupted = true;
+ resumeReplayAfterFailedSuccessiveHandoffLocked();
+ Thread.currentThread().interrupt();
+ throw new HandoffQuiescenceException("Interrupted while freezing Redis replay for successive handoff");
+ }
+ }
+ java.util.ArrayList pending = new java.util.ArrayList<>(deliveriesAfterReplay);
+ deliveriesAfterReplay.clear();
+ replayingHandoff = false;
+ replayGeneration++;
+ legacyLifecycle.notifyAll();
+ if (interrupted) Thread.currentThread().interrupt();
+ return pending;
+ }
+ }
+
+ /** Prepends a predecessor's still-unplayed replay FIFO before this standby's own overlap buffer. */
+ public void acceptReplayFromPreviousHandoff(java.util.List predecessorReplay) {
+ if (predecessorReplay == null || predecessorReplay.isEmpty()) return;
+ synchronized (legacyLifecycle) {
+ if (deliveriesAfterReplay.size() + replayDeliveriesInFlight + bufferedLegacyDeliveries.size()
+ + bufferedIdentifiedDeliveries.size() + predecessorReplay.size()
+ > MAX_REPLAY_HANDOFF_DELIVERIES)
+ throw new IllegalStateException("Redis replacement replay capacity is exhausted");
+ for (JsonEnvelope envelope : predecessorReplay) deliveriesAfterReplay.addLast(envelope);
+ }
+ }
+
+ /** Rolls back a not-yet-published predecessor transfer when its old listener cannot retire. */
+ public void removeReplayFromPreviousHandoff(java.util.List predecessorReplay) {
+ if (predecessorReplay == null || predecessorReplay.isEmpty()) return;
+ synchronized (legacyLifecycle) {
+ if (deliveriesAfterReplay.size() < predecessorReplay.size())
+ throw new IllegalStateException("Redis replacement replay transfer is incomplete");
+ for (JsonEnvelope expected : predecessorReplay) {
+ JsonEnvelope actual = deliveriesAfterReplay.removeFirst();
+ if (!java.util.Objects.equals(actual, expected))
+ throw new IllegalStateException("Redis replacement replay FIFO changed during handoff rollback");
+ }
+ }
+ }
+
+ /** Finalizes a successful transfer and releases callbacks to the promoted standby overlap. */
+ public void completeFrozenReplayTransfer() {
+ synchronized (legacyLifecycle) {
+ replayTransferFrozen = false;
+ legacyLifecycle.notifyAll();
+ }
+ }
+
+ /** Restores the frozen active FIFO when successor admission or retirement fails. */
+ public void restoreFrozenReplayAfterFailedSuccessiveHandoff(java.util.List replay) {
+ synchronized (legacyLifecycle) {
+ if (!deliveriesAfterReplay.isEmpty())
+ throw new IllegalStateException("Redis replay ownership changed while successive handoff was aborted");
+ deliveriesAfterReplay.addAll(replay);
+ resumeReplayAfterFailedSuccessiveHandoffLocked();
+ }
+ if (plugin != null) startReplayWorkerIfNeeded();
+ }
+
+ private void resumeReplayAfterFailedSuccessiveHandoffLocked() {
+ replayTransferFrozen = false;
+ retiredAfterHandoff = false;
+ if (!deliveriesAfterReplay.isEmpty()) replayingHandoff = true;
+ legacyLifecycle.notifyAll();
+ }
+
+ /** Prevents a listener that misses its shutdown deadline from dispatching duplicates. */
+ void fenceAfterHandoff() {
+ boolean interrupted = false;
+ long deadline = System.nanoTime() + HANDOFF_QUIESCE_TIMEOUT_NANOS;
+ synchronized (legacyLifecycle) {
+ retiredAfterHandoff = true;
+ replayGeneration++;
+ Thread worker = replayWorker;
+ if (worker != null) worker.interrupt();
+ replayWorker = null;
+ while (dispatchesInFlight > 0) {
+ long remaining = deadline - System.nanoTime();
+ if (remaining <= 0) break;
+ try {
+ TimeUnit.NANOSECONDS.timedWait(legacyLifecycle, remaining);
+ } catch (InterruptedException waitInterrupted) {
+ interrupted = true;
+ break;
+ }
+ }
+ if (dispatchesInFlight > 0) {
+ retiredAfterHandoff = false;
+ if (interrupted) Thread.currentThread().interrupt();
+ throw new HandoffQuiescenceException(
+ "Redis backend callbacks did not quiesce before handoff");
+ }
+ replayingHandoff = false;
+ deliveriesAfterReplay.clear();
+ replayBackpressureFailureLogged = false;
+ replayTaskOutstanding = false;
+ replayTaskGeneration = -1L;
+ replayDeliveriesInFlight = 0;
+ bufferedLegacyDeliveries.clear();
+ bufferedLegacyDeliveryBytes = 0;
+ bufferedIdentifiedDeliveries.clear();
+ bufferedIdentifiedDeliveryBytes = 0;
+ identifiedHandoffOverflowed = false;
+ legacyHandoffOverflowed = false;
+ }
+ if (interrupted) Thread.currentThread().interrupt();
+ }
+
+ private void bufferIdentifiedDelivery(JsonEnvelope envelope, String deliveryId) {
+ String signature = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.encode(envelope);
+ int encodedBytes = ProcessedVoteCache.legacyRedisDeliveryBytes(signature);
+ if (encodedBytes <= ProcessedVoteCache.MAX_LEGACY_REDIS_DELIVERY_BYTES
+ && bufferedIdentifiedDeliveries.size() < MAX_IDENTIFIED_HANDOFF_DELIVERIES
+ && handoffDeliveryCount() < MAX_REPLAY_HANDOFF_DELIVERIES
+ && bufferedIdentifiedDeliveryBytes <= ProcessedVoteCache.MAX_LEGACY_REDIS_TOTAL_BYTES - encodedBytes) {
+ bufferedIdentifiedDeliveries.add(new BufferedHandoffDelivery(
+ nextHandoffSequence++, envelope, deliveryId, true));
+ bufferedIdentifiedDeliveryBytes += encodedBytes;
+ } else {
+ if (plugin != null && !identifiedHandoffOverflowed)
+ plugin.getLogger().warning("Redis identified handoff buffer is full; aborting the staged handoff");
+ identifiedHandoffOverflowed = true;
+ }
+ }
+
+ /** Includes inherited replay plus staged overlap entries so rollback remains bounded. */
+ private int handoffDeliveryCount() {
+ return deliveriesAfterReplay.size() + replayDeliveriesInFlight
+ + bufferedIdentifiedDeliveries.size() + bufferedLegacyDeliveries.size();
+ }
+
+ private record BufferedHandoffDelivery(long sequence, JsonEnvelope envelope, String identity,
+ boolean identified) {}
+
@Override
- public void send(JsonEnvelope envelope) {
+ public boolean send(JsonEnvelope envelope) {
if (redisHandler != null) {
- redisHandler.publishEnvelope(plugin.getBungeeSettings().getRedisPrefix() + "VotingPlugin",
+ redisHandler.publishEnvelope(publishChannel,
VotingPluginWire.withRedisDeliveryId(envelope));
+ return true;
}
+ return false;
}
@Override
@@ -179,9 +891,28 @@ static DefaultJedisClientConfig buildValidationClientConfig(int database, String
@Override
public void close() {
+ cancelReplay();
closeListener(true);
}
+ private void cancelReplay() {
+ synchronized (legacyLifecycle) {
+ retiredAfterHandoff = true;
+ replayGeneration++;
+ replayingHandoff = false;
+ replayTransferFrozen = false;
+ deliveriesAfterReplay.clear();
+ replayBackpressureFailureLogged = false;
+ replayTaskOutstanding = false;
+ replayTaskGeneration = -1L;
+ replayDeliveriesInFlight = 0;
+ Thread worker = replayWorker;
+ replayWorker = null;
+ if (worker != null) worker.interrupt();
+ legacyLifecycle.notifyAll();
+ }
+ }
+
private void closeListener(boolean unregister) {
Thread thread = listenerThread;
if (redisHandler != null) {
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/SocketBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/SocketBackendProxyTransport.java
index a660c8874..da30de584 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/SocketBackendProxyTransport.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/SocketBackendProxyTransport.java
@@ -1,6 +1,9 @@
package com.bencodez.votingplugin.backendproxy.transport;
import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
import com.bencodez.simpleapi.encryption.EncryptionHandler;
import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope;
@@ -8,6 +11,7 @@
import com.bencodez.simpleapi.servercomm.sockets.ClientHandler;
import com.bencodez.simpleapi.servercomm.sockets.SocketHandler;
import com.bencodez.simpleapi.servercomm.sockets.SocketReceiver;
+import com.bencodez.simpleapi.servercomm.sockets.SocketServer;
import com.bencodez.votingplugin.VotingPluginMain;
import lombok.Getter;
@@ -19,6 +23,13 @@ public class SocketBackendProxyTransport implements BackendProxyTransport {
private ClientHandler clientHandler;
@Getter
private SocketHandler socketHandler;
+ private GlobalMessageHandler messageHandler;
+ private EncryptionHandler encryptionHandler;
+ private String bungeeHost;
+ private int bungeePort;
+ private String spigotHost;
+ private int spigotPort;
+ private boolean debug;
public SocketBackendProxyTransport(VotingPluginMain plugin) {
this.plugin = plugin;
@@ -26,32 +37,62 @@ public SocketBackendProxyTransport(VotingPluginMain plugin) {
@Override
public void start(GlobalMessageHandler messageHandler) {
- EncryptionHandler encryptionHandler = new EncryptionHandler(plugin.getName(),
+ this.messageHandler = messageHandler;
+ encryptionHandler = new EncryptionHandler(plugin.getName(),
new File(plugin.getDataFolder(), "secretkey.key"));
- clientHandler = new ClientHandler(plugin.getBungeeSettings().getBungeeServerHost(),
- plugin.getBungeeSettings().getBungeeServerPort(), encryptionHandler,
- plugin.getBungeeSettings().isBungeeDebug());
- socketHandler = new SocketHandler("vp-socket", plugin.getBungeeSettings().getSpigotServerHost(),
- plugin.getBungeeSettings().getSpigotServerPort(), encryptionHandler,
- plugin.getBungeeSettings().isBungeeDebug()) {
- @Override
- public void log(String str) {
- plugin.getLogger().info(str);
- }
- };
- socketHandler.add(new SocketReceiver() {
- @Override
- public void onReceiveEnvelope(JsonEnvelope envelope) {
- messageHandler.onMessage(envelope);
- }
- });
+ bungeeHost = plugin.getBungeeSettings().getBungeeServerHost();
+ bungeePort = plugin.getBungeeSettings().getBungeeServerPort();
+ spigotHost = plugin.getBungeeSettings().getSpigotServerHost();
+ spigotPort = plugin.getBungeeSettings().getSpigotServerPort();
+ debug = plugin.getBungeeSettings().isBungeeDebug();
+ startConnections();
+ }
+
+ private void startConnections() {
+ clientHandler = new ClientHandler(bungeeHost, bungeePort, encryptionHandler, debug);
+ try {
+ verifyListenerPortAvailable();
+ socketHandler = new SocketHandler("vp-socket", spigotHost, spigotPort, encryptionHandler, debug) {
+ @Override
+ public void log(String str) {
+ plugin.getLogger().info(str);
+ }
+ };
+ socketHandler.add(new SocketReceiver() {
+ @Override
+ public void onReceiveEnvelope(JsonEnvelope envelope) {
+ messageHandler.onMessage(envelope);
+ }
+ });
+ } catch (RuntimeException failure) {
+ clientHandler.stopConnection();
+ clientHandler = null;
+ throw failure;
+ }
+ }
+
+ /**
+ * SocketHandler logs and closes itself when its constructor cannot bind. Probe
+ * first so replacement validation cannot mistake that swallowed failure for a
+ * live listener.
+ */
+ private void verifyListenerPortAvailable() {
+ try (ServerSocket probe = new ServerSocket()) {
+ probe.setReuseAddress(false);
+ probe.bind(new InetSocketAddress(spigotHost, spigotPort));
+ } catch (IOException unavailable) {
+ throw new IllegalStateException("Socket backend proxy listener is unavailable at " + spigotHost + ":"
+ + spigotPort, unavailable);
+ }
}
@Override
- public void send(JsonEnvelope envelope) {
+ public boolean send(JsonEnvelope envelope) {
if (clientHandler != null) {
clientHandler.sendEnvelope(envelope);
+ return true;
}
+ return false;
}
@Override
@@ -63,13 +104,64 @@ public void validate() {
@Override
public void close() {
- if (socketHandler != null) {
- socketHandler.closeConnection();
- socketHandler = null;
+ close(false);
+ }
+
+ @Override
+ public void prepareForReplacement() {
+ close(true);
+ }
+
+ private void close(boolean strict) {
+ RuntimeException failure = null;
+ try {
+ closeSocketListener();
+ } catch (RuntimeException listenerFailure) {
+ failure = listenerFailure;
}
- if (clientHandler != null) {
- clientHandler.stopConnection();
- clientHandler = null;
+ ClientHandler closingClient = clientHandler;
+ clientHandler = null;
+ if (closingClient != null) try {
+ closingClient.stopConnection();
+ } catch (RuntimeException clientFailure) {
+ if (failure == null) failure = clientFailure;
+ else failure.addSuppressed(clientFailure);
+ }
+ if (failure == null) return;
+ if (strict) throw failure;
+ if (plugin != null && plugin.getLogger() != null) {
+ plugin.getLogger().warning("Socket backend proxy transport did not stop cleanly");
+ plugin.debug(failure);
+ }
+ }
+
+ private void closeSocketListener() {
+ SocketHandler closing = socketHandler;
+ socketHandler = null;
+ if (closing == null) return;
+ SocketServer server = closing.getServer();
+ closing.closeConnection();
+ if (server == null) return;
+ try {
+ server.join(1000L);
+ if (server.isAlive()) {
+ throw new IllegalStateException("Socket backend proxy listener did not stop before replacement");
+ }
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException("Interrupted while retiring the socket backend proxy listener", interrupted);
+ } finally {
+ // SocketServer's accept-error recovery can race close() and re-bind once
+ // before it observes its stopped flag. Always close the final socket, even
+ // after a timeout or interruption leaves the worker incomplete.
+ server.close();
+ }
+ }
+
+ public void restoreAfterFailedReplacement() {
+ if (messageHandler == null || encryptionHandler == null) {
+ throw new IllegalStateException("Socket backend proxy transport cannot be restored before startup");
}
+ startConnections();
}
}
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/config/BungeeSettings.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/config/BungeeSettings.java
index b9f8b1afc..5c8a05ab7 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/config/BungeeSettings.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/config/BungeeSettings.java
@@ -88,7 +88,11 @@ public class BungeeSettings extends YMLFile {
@ConfigDataInt(path = "BungeeServer.Port")
@Getter
- private int bungeeServerPort = 1297;
+ private int bungeeServerPort = 1297;
+
+ @ConfigDataString(path = "HTTP.ConnectionCode")
+ @Getter
+ private String httpConnectionCode = "";
@ConfigDataBoolean(path = "PerServerPoints")
@Getter
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java
index f4722b261..18ef84393 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java
@@ -36,6 +36,7 @@
import org.bukkit.configuration.InvalidConfigurationException;
import org.bukkit.configuration.file.YamlConfiguration;
+import com.bencodez.votingplugin.backendproxy.transport.HttpBackendProxyTransport;
import com.bencodez.votingplugin.proxy.BungeeMethod;
import com.bencodez.votingplugin.util.DurableFiles;
@@ -59,7 +60,7 @@ public final class BackendConfigurationService {
"WaitUntilVoteDelay", "PermissionToView", "IgnoreCanVote", "VoteDelayDailyHour", "VoteDelayMin",
"GiveOffline");
private static final Pattern COMMENT_SECRET = Pattern.compile(
- "(?i)([\"']?\\b(?:[\\w-]*(?:password|secret|user(?:name)?)[\\w-]*|token|api[ _.-]?key|authorization|[\\w.-]*webhook[ _.-]?url)"
+ "(?i)([\"']?\\b(?:[\\w-]*(?:password|secret|user(?:name)?)[\\w-]*|token|connection[ _.-]?code|api[ _.-]?key|authorization|[\\w.-]*webhook[ _.-]?url)"
+ "\\b[\"']?\\s*[:=]\\s*)(.*)$");
private static final Pattern SECRET_PATH_URL = Pattern.compile("(?i)([\"']?\\burl\\b[\"']?\\s*[:=]\\s*)(.*)$");
private static final Pattern BLOCK_SCALAR_INDICATOR = Pattern.compile("[|>](?:[+-][1-9]?|[1-9][+-]?)?");
@@ -289,6 +290,7 @@ private ApplyResult apply(String fileName, String proposedContent, String expect
Path staging = Files.createTempFile(target.getParent(), ".control-", ".yml");
Path backupStaging = Files.createTempFile(target.getParent(), ".control-backup-", ".yml");
boolean installed = false;
+ boolean reloadAttempted = false;
try {
Files.writeString(staging, preview.resolvedContent(), StandardCharsets.UTF_8,
StandardOpenOption.TRUNCATE_EXISTING);
@@ -305,6 +307,7 @@ private ApplyResult apply(String fileName, String proposedContent, String expect
installed = true;
throw published;
}
+ reloadAttempted = true;
applyAction.run(fileName);
String applied = readRaw(target, false);
String installedRevision = revision(preview.resolvedContent());
@@ -337,7 +340,7 @@ private ApplyResult apply(String fileName, String proposedContent, String expect
failure.addSuppressed(rollbackFailure);
}
}
- throw new ApplyFailureException(rolledBack, failure);
+ throw new ApplyFailureException(rolledBack, reloadAttempted, failure);
} finally {
Files.deleteIfExists(staging);
Files.deleteIfExists(backupStaging);
@@ -374,6 +377,7 @@ private synchronized ApplyResult applyNamedReward(String fileName, String propos
String rollbackStaging = null;
String backup = name + ".control-backup";
boolean installed = false;
+ boolean reloadAttempted = false;
try {
writeRaw(rewards, staging, preview.resolvedContent(), true);
rejectSymbolicBackup(rewards, backup);
@@ -385,6 +389,7 @@ private synchronized ApplyResult applyNamedReward(String fileName, String propos
rewards.move(Path.of(staging), rewards, Path.of(name));
installed = true;
forcePinnedRewardDirectory(rewards, directoryKey);
+ reloadAttempted = true;
applyAction.runNamedReward(fileName, preview.resolvedContent(),
() -> requireCurrentRewardDirectory(directoryKey));
requireCurrentRewardDirectory(directoryKey);
@@ -408,9 +413,10 @@ private synchronized ApplyResult applyNamedReward(String fileName, String propos
if (!revision(readRaw(rewards, name, false)).equals(revision(preview.resolvedContent()))) {
throw new IOException("Managed configuration changed while rollback was staged");
}
- rewards.move(Path.of(rollbackStaging), rewards, Path.of(name));
- forcePinnedRewardDirectory(rewards, directoryKey);
- applyAction.runNamedReward(fileName, current,
+ rewards.move(Path.of(rollbackStaging), rewards, Path.of(name));
+ forcePinnedRewardDirectory(rewards, directoryKey);
+ reloadAttempted = true;
+ applyAction.runNamedReward(fileName, current,
() -> requireCurrentRewardDirectory(directoryKey));
requireCurrentRewardDirectory(directoryKey);
rolledBack = true;
@@ -418,7 +424,7 @@ private synchronized ApplyResult applyNamedReward(String fileName, String propos
failure.addSuppressed(rollbackFailure);
}
}
- throw new ApplyFailureException(rolledBack, failure);
+ throw new ApplyFailureException(rolledBack, reloadAttempted, failure);
} finally {
deleteIfPresent(rewards, staging);
deleteIfPresent(rewards, backupStaging);
@@ -1061,6 +1067,11 @@ private void validateProxyMethod(BungeeMethod method, YamlConfiguration settings
case SOCKETS:
configuredHostAndPort(settings, "BungeeServer.Host", "BungeeServer.Port", 1297, "BungeeServer");
break;
+ case HTTP:
+ String connectionCode = settings.getString("HTTP.ConnectionCode", "");
+ try { HttpBackendProxyTransport.validateConfiguration(dataDirectory.resolve("http"), server, connectionCode); }
+ catch (IllegalStateException invalid) { throw new IllegalArgumentException(invalid.getMessage(), invalid); }
+ break;
case MYSQL:
try {
YamlConfiguration main = parse(readRaw(resolve("Config.yml"), false));
@@ -1275,24 +1286,31 @@ private static boolean rewardFileName(String name) {
}
private static String readRaw(Path path, boolean allowMissing) throws IOException {
- if (allowMissing && !Files.exists(path, LinkOption.NOFOLLOW_LINKS)) return "";
+ if (!Files.exists(path, LinkOption.NOFOLLOW_LINKS)) {
+ if (allowMissing) return "";
+ throw new java.nio.file.NoSuchFileException(path.toString());
+ }
if (!Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS)) {
- throw new IOException("configuration file is missing or too large");
+ throw new ConfigurationReadException(ConfigurationReadFailure.UNSAFE,
+ "configuration file is not a regular file");
}
byte[] bytes;
// Keep the no-follow check attached to the opened file. A separate Files.size/readAllBytes
// would follow a link swapped in after the regular-file check.
try (SeekableByteChannel channel = Files.newByteChannel(path,
Set.of(StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS))) {
- if (channel.size() > MAX_CONTENT_BYTES) throw new IOException("configuration file is missing or too large");
+ if (channel.size() > MAX_CONTENT_BYTES) throw new ConfigurationReadException(
+ ConfigurationReadFailure.TOO_LARGE, "configuration file exceeds the managed size limit");
bytes = Channels.newInputStream(channel).readNBytes(MAX_CONTENT_BYTES + 1);
- if (bytes.length > MAX_CONTENT_BYTES) throw new IOException("configuration file is missing or too large");
+ if (bytes.length > MAX_CONTENT_BYTES) throw new ConfigurationReadException(
+ ConfigurationReadFailure.TOO_LARGE, "configuration file exceeds the managed size limit");
}
try {
return StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT).decode(java.nio.ByteBuffer.wrap(bytes)).toString();
} catch (CharacterCodingException e) {
- throw new IOException("configuration file is not valid UTF-8", e);
+ throw new ConfigurationReadException(ConfigurationReadFailure.INVALID_ENCODING,
+ "configuration file is not valid UTF-8", e);
}
}
@@ -1478,7 +1496,7 @@ private static List restoreCommentSecrets(List proposed, List CAPABILITIES = Set.of("config.files.v1", "config.file-comments.v1",
"config.quick-setup.v1", "config.quick-setup.v2", "config.vote-sites-sync.v1",
- "config.proxy-method.v1", "config.reward-files.v1", "data.inspect.v1");
+ "config.proxy-method.v1", "config.proxy-method.v2", "config.reward-files.v1", "data.inspect.v1");
+ private static final Set DEPLOYMENT_TASK_FIELDS = Set.of("deploymentId", "artifactId", "sha256", "size", "attemptId");
private final VotingPluginMain plugin;
private final Path dataDirectory;
@@ -65,27 +70,33 @@ public final class BackendControlConnector implements AutoCloseable {
private final HttpClient http;
private final BackendConfigurationService configurations;
private final ControlInspectionService inspections;
+ private final PluginDeploymentService deployments;
+ private final boolean directLocalDeploymentEndpoint;
private final UUID sessionId = UUID.randomUUID();
private final Map completed = new LinkedHashMap<>();
private final boolean recovering;
private final AtomicBoolean running = new AtomicBoolean();
private final AtomicBoolean inspecting = new AtomicBoolean();
private final Object operationLifecycle = new Object();
+ private final AtomicReference pendingBackendProxyRollback = new AtomicReference<>();
private final Object journalLifecycle = new Object();
private volatile boolean closed;
private volatile boolean registered;
private volatile boolean operationsAccepted;
private volatile boolean quickSetupsAccepted;
+ private volatile boolean proxyMethodV2Accepted;
private volatile boolean votePartySetupsAccepted;
private volatile boolean voteSitesSyncAccepted;
private volatile boolean rewardFilesAccepted;
private volatile boolean inspectionsAccepted;
+ private volatile boolean deploymentsAccepted;
private volatile int inspectionFailures;
private volatile long inspectionRetryAtNanos;
private volatile int failures;
private volatile ScheduledFuture scheduled;
private volatile ScheduledFuture operationPolling;
private volatile ScheduledFuture inspectionPolling;
+ private volatile ScheduledFuture deploymentPolling;
private volatile Future activeReload;
private volatile CompletableFuture activeOperation;
@@ -128,23 +139,168 @@ private BackendControlConnector(VotingPluginMain plugin, Path dataDirectory, Set
}
});
inspections = new ControlInspectionService(plugin);
+ directLocalDeploymentEndpoint = HostedControlManager.isDirectLocalEndpoint(
+ settings.endpoint().toString(), hostedConfiguration);
+ PluginDeploymentService prepared = null;
+ boolean deploymentEndpointAllowed = PluginDeploymentService.credentialEndpointAllowed(
+ settings.endpoint(), directLocalDeploymentEndpoint);
+ if (!recovering && deploymentEndpointAllowed) {
+ try {
+ prepared = PluginDeploymentService.backend(plugin.getServer().getUpdateFolderFile().toPath(),
+ plugin.getLoadedPluginJarFile().toPath());
+ } catch (Exception failure) {
+ plugin.getLogger().warning("[Control] Plugin deployment staging is unavailable; capability not advertised");
+ }
+ } else if (!recovering && !deploymentEndpointAllowed) {
+ plugin.getLogger().warning("[Control] Plugin deployment staging requires HTTPS unless Control is hosted directly on this node");
+ }
+ deployments = prepared;
}
private void reloadConfiguration(String fileName, String expectedContent) throws Exception {
- reloadConfiguration(fileName, expectedContent, null);
+ reloadConfiguration(fileName, expectedContent, null, false);
}
private void reloadConfiguration(String fileName, String expectedContent,
BackendConfigurationService.NamedRewardGuard guard) throws Exception {
- reloadOnServerThread(() -> {
- // Publication happens off-thread, but the plugin reads Rewards by path
- // on this owner thread. Recheck the pinned identity at that boundary.
- if (guard != null) guard.verify();
- plugin.reloadFromControl();
- if (guard != null) guard.verify();
- if (BackendConfigurationService.managedRewardFile(fileName)) verifyNamedRewardLoaded(fileName, expectedContent);
- if ("BungeeSettings.yml".equals(fileName)) plugin.restartBackendProxyHandler();
- });
+ reloadConfiguration(fileName, expectedContent, guard, false);
+ }
+
+ /** Reuses the split restart lifecycle while retaining proxy-method's narrow reload scope. */
+ private void reloadConfiguration(String fileName, String expectedContent,
+ BackendConfigurationService.NamedRewardGuard guard, boolean proxyMethodOnly) throws Exception {
+ finishPendingBackendProxyRollback();
+ long validationDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(29);
+ AtomicBoolean preparationAbandoned = new AtomicBoolean();
+ AtomicInteger preparationState = new AtomicInteger();
+ CountDownLatch preparationSettled = new CountDownLatch(1);
+ CountDownLatch rollbackAbortSettled = new CountDownLatch(1);
+ AtomicReference preparedRestart = new AtomicReference<>();
+ Future preparation;
+ synchronized (operationLifecycle) {
+ if (closed) throw new IllegalStateException("Bukkit Control connector is stopping");
+ preparation = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> {
+ try {
+ if (!preparationState.compareAndSet(0, 1)) return null;
+ if (guard != null) guard.verify();
+ if (!proxyMethodOnly) plugin.reloadFromControl();
+ if (guard != null) guard.verify();
+ if (BackendConfigurationService.managedRewardFile(fileName)) {
+ verifyNamedRewardLoaded(fileName, expectedContent);
+ }
+ VotingPluginMain.BackendProxyRestart prepared;
+ try {
+ if (!"BungeeSettings.yml".equals(fileName)) prepared = null;
+ else if (proxyMethodOnly) prepared = plugin.prepareBackendProxyMethodRestartFromControl();
+ else prepared = plugin.prepareBackendProxyHandlerRestart();
+ } catch (VotingPluginMain.BackendProxyRestartPreparationException failure) {
+ preparedRestart.set(failure.restart());
+ throw failure;
+ }
+ preparedRestart.set(prepared);
+ return prepared;
+ } finally {
+ try {
+ VotingPluginMain.BackendProxyRestart prepared = preparedRestart.get();
+ if (preparationAbandoned.get()) {
+ // The Control worker performs transport restoration after this
+ // Bukkit callback has marked the staged handler unavailable.
+ if (prepared != null && prepared.requiresWorkerRollback()) {
+ plugin.requestBackendProxyHandlerRestartAbandonment(prepared);
+ } else if (prepared != null) {
+ try {
+ plugin.abortBackendProxyHandlerRestart(prepared);
+ } finally { rollbackAbortSettled.countDown(); }
+ }
+ }
+ } finally {
+ preparationState.set(2);
+ preparationSettled.countDown();
+ }
+ }
+ });
+ activeReload = preparation;
+ }
+ VotingPluginMain.BackendProxyRestart restart = null;
+ Future publication = null;
+ try {
+ restart = preparation.get(remaining(validationDeadline), TimeUnit.NANOSECONDS);
+ if (restart == null) return;
+ // Network enrollment/readiness is deliberately awaited on this Control worker,
+ // never on Bukkit's primary thread.
+ plugin.validateBackendProxyHandlerRestart(restart, validationDeadline);
+ VotingPluginMain.BackendProxyRestart prepared = restart;
+ publication = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> {
+ plugin.completeBackendProxyHandlerRestart(prepared);
+ return null;
+ });
+ synchronized (operationLifecycle) {
+ if (closed) publication.cancel(true);
+ activeReload = publication;
+ }
+ publication.get(remaining(validationDeadline), TimeUnit.NANOSECONDS);
+ } catch (Exception failure) {
+ // Timed-out Bukkit work must not remain queued ahead of configuration rollback.
+ preparationAbandoned.set(true);
+ boolean abandonedBeforePreparation = preparationState.compareAndSet(0, 3);
+ preparation.cancel(false);
+ boolean publicationCommitted = publication != null && restart != null
+ && !plugin.requestBackendProxyHandlerRestartAbandonment(restart);
+ if (publication != null) publication.cancel(false);
+ // A publication that won the synchronized commit race is the successful
+ // runtime state; rolling its YAML back would create a split-brain config.
+ if (publicationCommitted) return;
+ PendingBackendProxyRollback pendingRollback = null;
+ if (restart == null && !abandonedBeforePreparation) {
+ pendingRollback = new PendingBackendProxyRollback(preparationSettled, rollbackAbortSettled, preparedRestart);
+ pendingBackendProxyRollback.set(pendingRollback);
+ try {
+ if (!preparationSettled.await(40, TimeUnit.SECONDS))
+ throw new java.util.concurrent.TimeoutException(
+ "Bukkit configuration preparation did not settle during rollback");
+ } catch (Exception preparationFailure) {
+ failure.addSuppressed(preparationFailure);
+ throw failure;
+ }
+ }
+ if (restart == null) restart = preparedRestart.get();
+ if (restart != null) {
+ if (pendingRollback == null) {
+ pendingRollback = new PendingBackendProxyRollback(
+ preparationSettled, rollbackAbortSettled, preparedRestart);
+ pendingBackendProxyRollback.set(pendingRollback);
+ }
+ VotingPluginMain.BackendProxyRestart prepared = restart;
+ try {
+ if (prepared.requiresWorkerRollback()) {
+ // This worker owns the exclusive socket/MQTT teardown and restoration.
+ // Bukkit state was already fenced by requestBackendProxyHandlerRestartAbandonment.
+ plugin.abortBackendProxyHandlerRestart(prepared);
+ rollbackAbortSettled.countDown();
+ } else {
+ Future abort = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> {
+ try { plugin.abortBackendProxyHandlerRestart(prepared); }
+ finally { rollbackAbortSettled.countDown(); }
+ return null;
+ });
+ abort.get(5, TimeUnit.SECONDS);
+ }
+ finishPendingBackendProxyRollback();
+ } catch (Exception cleanupFailure) {
+ rollbackAbortSettled.countDown();
+ failure.addSuppressed(cleanupFailure);
+ }
+ }
+ if (pendingRollback != null && restart == null) {
+ rollbackAbortSettled.countDown();
+ pendingBackendProxyRollback.compareAndSet(pendingRollback, null);
+ }
+ throw failure;
+ } finally {
+ synchronized (operationLifecycle) {
+ if (activeReload == preparation || activeReload == publication) activeReload = null;
+ }
+ }
}
private void reloadConfiguration(String fileName) throws Exception {
@@ -194,26 +350,40 @@ private static boolean sameConfigurationValues(ConfigurationSection source, Conf
}
private void reloadProxyMethod(String ignored) throws Exception {
- reloadOnServerThread(plugin::reloadBackendProxyMethodFromControl);
- }
-
- private void reloadOnServerThread(ThrowingRunnable action) throws Exception {
- Future reload;
- synchronized (operationLifecycle) {
- if (closed) throw new IllegalStateException("Bukkit Control connector is stopping");
- reload = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { action.run(); return null; });
- activeReload = reload;
- }
- try {
- reload.get(30, TimeUnit.SECONDS);
- } finally {
- synchronized (operationLifecycle) {
- if (activeReload == reload) activeReload = null;
- }
+ // Proxy-method APPLY can select HTTP or Redis. Reuse the BungeeSettings
+ // split lifecycle so Bukkit only prepares/publishes while validation and
+ // bounded transport handoff waits remain on this connector worker.
+ reloadConfiguration("BungeeSettings.yml", null, null, true);
+ }
+
+ private void finishPendingBackendProxyRollback() throws Exception {
+ PendingBackendProxyRollback pending = pendingBackendProxyRollback.get();
+ if (pending == null) return;
+ if (!pending.preparationSettled().await(40, TimeUnit.SECONDS))
+ throw new java.util.concurrent.TimeoutException(
+ "Previous Bukkit configuration preparation is still rolling back");
+ VotingPluginMain.BackendProxyRestart restart = pending.preparedRestart().get();
+ if (restart != null) {
+ if (!pending.rollbackAbortSettled().await(40, TimeUnit.SECONDS))
+ throw new java.util.concurrent.TimeoutException(
+ "Previous Bukkit configuration abort is still pending");
+ // Keep credential-journal I/O off Bukkit's primary thread and finish it
+ // before the configuration service starts its automatic backup reload.
+ plugin.awaitBackendProxyHandlerRollback(restart,
+ System.nanoTime() + TimeUnit.SECONDS.toNanos(40));
}
+ pendingBackendProxyRollback.compareAndSet(pending, null);
}
- @FunctionalInterface private interface ThrowingRunnable { void run() throws Exception; }
+ private record PendingBackendProxyRollback(CountDownLatch preparationSettled,
+ CountDownLatch rollbackAbortSettled,
+ AtomicReference preparedRestart) { }
+
+ private static long remaining(long deadlineNanos) throws java.util.concurrent.TimeoutException {
+ long remaining = deadlineNanos - System.nanoTime();
+ if (remaining <= 0L) throw new java.util.concurrent.TimeoutException("Bukkit configuration reload timed out");
+ return remaining;
+ }
public static BackendControlConnector create(VotingPluginMain plugin) throws IOException {
Path root = plugin.getDataFolder().toPath().toAbsolutePath().normalize();
@@ -268,6 +438,76 @@ public void start() {
OPERATION_POLL_MILLIS, OPERATION_POLL_MILLIS, TimeUnit.MILLISECONDS);
inspectionPolling = inspectionExecutor.scheduleWithFixedDelay(this::pollInspections,
OPERATION_POLL_MILLIS, OPERATION_POLL_MILLIS, TimeUnit.MILLISECONDS);
+ if (deployments != null) {
+ deploymentPolling = executor.scheduleWithFixedDelay(this::pollDeployments,
+ OPERATION_POLL_MILLIS, OPERATION_POLL_MILLIS, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ /** Deployment is a separate leased lane; its large I/O never runs on Bukkit's primary thread. */
+ private void pollDeployments() {
+ if (closed || deployments == null || !registered || failures != 0 || !deploymentsAccepted
+ || !running.compareAndSet(false, true)) return;
+ CompletableFuture operation = new CompletableFuture<>();
+ synchronized (operationLifecycle) {
+ if (closed) {
+ running.set(false);
+ return;
+ }
+ activeOperation = operation;
+ }
+ try {
+ claimAndDeploy();
+ } catch (Exception failure) {
+ registered = false;
+ failures = Math.min(30, failures + 1);
+ if (failures == 1 || failures % 10 == 0) {
+ plugin.getLogger().warning("[Control] Bukkit deployment polling unavailable; VotingPlugin remains active");
+ }
+ ScheduledFuture heartbeat = scheduled;
+ if (heartbeat != null) heartbeat.cancel(false);
+ if (!closed) schedule(Math.min(TimeUnit.MINUTES.toMillis(5),
+ 1000L << Math.min(failures - 1, 8)));
+ } finally {
+ operation.complete(null);
+ synchronized (operationLifecycle) {
+ if (activeOperation == operation) activeOperation = null;
+ }
+ running.set(false);
+ }
+ }
+
+ private void claimAndDeploy() throws Exception {
+ JsonObject body = new JsonObject();
+ body.addProperty("sessionId", sessionId.toString());
+ Response response = send("POST", "/api/v1/nodes/" + settings.nodeId() + "/deployments", body);
+ if (response.status() == 204 || closed) return;
+ JsonObject claimed = requireObject(response, 200);
+ PluginDeploymentService.Task task = deploymentTask(claimed);
+ PluginDeploymentService.Result result = deployments.deploy(task, settings.endpoint(), directLocalDeploymentEndpoint,
+ settings.nodeId(), sessionId, credential, http, Duration.ofMillis(settings.requestTimeoutMillis()),
+ () -> !closed);
+ if (closed) return;
+ JsonObject submitted = new JsonObject();
+ submitted.addProperty("sessionId", sessionId.toString());
+ submitted.addProperty("success", result.success());
+ submitted.addProperty("code", result.code());
+ submitted.addProperty("message", boundedResultMessage(result.message()));
+ submitted.addProperty("attemptId", task.attemptId().toString());
+ requireObject(send("POST", "/api/v1/nodes/" + settings.nodeId() + "/deployments/" + task.deploymentId()
+ + "/result", submitted), 200);
+ }
+
+ static PluginDeploymentService.Task deploymentTask(JsonObject task) {
+ try {
+ if (task == null || task.size() != DEPLOYMENT_TASK_FIELDS.size()
+ || !DEPLOYMENT_TASK_FIELDS.containsAll(task.keySet())) throw new IllegalArgumentException();
+ return new PluginDeploymentService.Task(UUID.fromString(string(task, "deploymentId")),
+ string(task, "artifactId"), string(task, "sha256"), Long.parseLong(string(task, "size")),
+ UUID.fromString(string(task, "attemptId")));
+ } catch (RuntimeException failure) {
+ throw new IllegalArgumentException("deployment task is invalid");
+ }
}
/** Polls the separately negotiated read-only lane on the connector worker. */
@@ -387,12 +627,15 @@ private void cycle() {
}
operationsAccepted = negotiatedCapability(node, "config.files.v1", operationsAccepted);
quickSetupsAccepted = negotiatedCapability(node, "config.quick-setup.v1", quickSetupsAccepted);
+ proxyMethodV2Accepted = negotiatedCapability(node, "config.proxy-method.v2", proxyMethodV2Accepted);
votePartySetupsAccepted = negotiatedCapability(node, "config.quick-setup.v2", votePartySetupsAccepted);
voteSitesSyncAccepted = negotiatedCapability(node, "config.vote-sites-sync.v1", voteSitesSyncAccepted);
rewardFilesAccepted = configurations.supportsNamedRewardFiles()
&& negotiatedCapability(node, "config.reward-files.v1", rewardFilesAccepted);
boolean inspectionsWereAccepted = inspectionsAccepted;
inspectionsAccepted = negotiatedCapability(node, "data.inspect.v1", inspectionsAccepted);
+ deploymentsAccepted = deployments != null
+ && negotiatedCapability(node, PluginDeploymentService.CAPABILITY, deploymentsAccepted);
if (inspectionsAccepted && !inspectionsWereAccepted) {
inspectionFailures = 0;
inspectionRetryAtNanos = 0;
@@ -445,6 +688,7 @@ private JsonObject register() throws Exception {
voteSitesSyncAccepted = false;
rewardFilesAccepted = false;
inspectionsAccepted = false;
+ deploymentsAccepted = false;
JsonObject body = sessionBody();
body.addProperty("nodeId", settings.nodeId());
body.addProperty("displayName", settings.nodeId());
@@ -455,13 +699,13 @@ private JsonObject register() throws Exception {
.map(installed -> installed.getDescription().getName()).filter(name -> name != null && !name.isBlank())
.distinct().sorted(String.CASE_INSENSITIVE_ORDER).limit(128).forEach(detectedPlugins::add);
body.add("detectedPlugins", detectedPlugins);
- addCapabilities(body, configurations.supportsNamedRewardFiles());
+ addCapabilities(body, configurations.supportsNamedRewardFiles(), deployments != null);
return requireObject(send("POST", "/api/v1/nodes/register", body), 200, 201);
}
private JsonObject heartbeat() throws Exception {
JsonObject body = sessionBody();
- addCapabilities(body, configurations.supportsNamedRewardFiles());
+ addCapabilities(body, configurations.supportsNamedRewardFiles(), deployments != null);
Response response = send("PUT", "/api/v1/nodes/" + settings.nodeId() + "/heartbeat", body);
if (response.status() == 404) {
registered = false;
@@ -711,8 +955,9 @@ private TaskResult execute(UUID operationId, JsonObject task) {
} catch (BackendConfigurationService.StaleRevisionException e) {
return TaskResult.failure("STALE_REVISION", "Configuration changed after preview");
} catch (BackendConfigurationService.ApplyFailureException e) {
- logConfigurationFailure("reload", e);
- return TaskResult.failure("RELOAD_FAILED", reloadFailureMessage(e), e.rolledBack());
+ logConfigurationFailure(e.reloadAttempted() ? "reload" : "write", e);
+ return TaskResult.failure(e.reloadAttempted() ? "RELOAD_FAILED" : "WRITE_FAILED",
+ applyFailureMessage(e), e.rolledBack());
} catch (IllegalArgumentException e) {
return TaskResult.failure("VALIDATION_ERROR", e.getMessage());
} catch (Exception e) {
@@ -732,16 +977,41 @@ private void logConfigurationFailure(String action, Throwable failure) {
}
static String operationFailureMessage(String type, Throwable failure) {
- if ("READ".equals(type) && failure instanceof IOException) {
+ if ("READ".equals(type)) {
+ if (hasCause(failure, java.nio.file.NoSuchFileException.class)) {
+ return "The managed configuration file does not exist on this backend";
+ }
+ if (hasCause(failure, java.nio.file.AccessDeniedException.class)
+ || hasCause(failure, SecurityException.class)) {
+ return "The managed configuration file is not readable by the backend";
+ }
+ BackendConfigurationService.ConfigurationReadException readFailure = cause(
+ failure, BackendConfigurationService.ConfigurationReadException.class);
+ if (readFailure != null) return switch (readFailure.failure()) {
+ case TOO_LARGE -> "The managed configuration file exceeds the 512 KiB limit";
+ case INVALID_ENCODING -> "The managed configuration file is not valid UTF-8";
+ case UNSAFE -> "The managed configuration path is not a safe regular file";
+ };
return "Configuration file is unavailable or unreadable";
}
- if ("READ".equals(type)) return "Configuration read failed; see the backend log";
- if ("PREVIEW".equals(type)) return "Configuration preview failed; see the backend log";
- return "Configuration apply failed; see the backend log";
+ if ("PREVIEW".equals(type)) return "The backend could not prepare a configuration preview";
+ return "The backend could not apply the managed configuration";
}
- static String reloadFailureMessage(Throwable ignored) {
- return "Configuration reload failed; see the backend log";
+ static String applyFailureMessage(BackendConfigurationService.ApplyFailureException failure) {
+ if (!failure.reloadAttempted()) {
+ return failure.rolledBack() ? "Configuration write failed; the previous file was restored"
+ : "Configuration write failed before runtime reload";
+ }
+ String reason = hasCause(failure, java.util.concurrent.TimeoutException.class)
+ || hasCause(failure, java.net.SocketTimeoutException.class)
+ ? "the configured transport did not become ready before its deadline"
+ : hasCause(failure, java.net.ConnectException.class)
+ ? "the configured transport endpoint was unavailable"
+ : "the backend runtime rejected the new configuration";
+ return failure.rolledBack()
+ ? "Runtime reload failed because " + reason + "; the previous file was restored"
+ : "Runtime reload failed because " + reason + "; automatic rollback did not complete";
}
static String operationFailureCode(String type) {
@@ -749,11 +1019,34 @@ static String operationFailureCode(String type) {
}
static String operationFailureCode(String type, Throwable failure) {
+ if ("READ".equals(type) && hasCause(failure, java.nio.file.NoSuchFileException.class)) {
+ return "CONFIGURATION_MISSING";
+ }
+ if ("READ".equals(type) && (hasCause(failure, java.nio.file.AccessDeniedException.class)
+ || hasCause(failure, SecurityException.class))) return "CONFIGURATION_UNREADABLE";
+ BackendConfigurationService.ConfigurationReadException readFailure = cause(
+ failure, BackendConfigurationService.ConfigurationReadException.class);
+ if ("READ".equals(type) && readFailure != null) return switch (readFailure.failure()) {
+ case TOO_LARGE -> "CONFIGURATION_TOO_LARGE";
+ case INVALID_ENCODING -> "CONFIGURATION_INVALID_ENCODING";
+ case UNSAFE -> "CONFIGURATION_UNSAFE";
+ };
if ("READ".equals(type)) return "READ_FAILED";
if ("PREVIEW".equals(type)) return "PREVIEW_FAILED";
return "APPLY_FAILED";
}
+ private static boolean hasCause(Throwable failure, Class type) {
+ return cause(failure, type) != null;
+ }
+
+ private static T cause(Throwable failure, Class type) {
+ for (Throwable current = failure; current != null; current = current.getCause()) {
+ if (type.isInstance(current)) return type.cast(current);
+ }
+ return null;
+ }
+
private TaskResult executeFile(UUID operationId, String type, JsonObject configuration, JsonObject task)
throws IOException {
String fileName = string(configuration, "fileName");
@@ -788,6 +1081,11 @@ private TaskResult executeQuick(UUID operationId, String type, JsonObject config
throws IOException {
String preset = string(configuration, "preset");
Map options = options(configuration.getAsJsonObject("options"));
+ if ("proxy-method".equals(preset)
+ && !proxyMethodApplyCapabilityAccepted(options.getOrDefault("method", "PLUGINMESSAGING"),
+ proxyMethodV2Accepted)) {
+ return TaskResult.failure("UNSUPPORTED_TASK", "The HTTP proxy method capability was not negotiated");
+ }
if (!quickSetupCapabilityAccepted(preset, quickSetupsAccepted, votePartySetupsAccepted,
voteSitesSyncAccepted, options)) {
return TaskResult.failure("UNSUPPORTED_TASK", "The required quick setup capability was not negotiated");
@@ -853,6 +1151,10 @@ static boolean quickSetupCapabilityAccepted(String preset, boolean quickSetupsAc
voteSitesSyncAccepted);
}
+ static boolean proxyMethodApplyCapabilityAccepted(String method, boolean proxyMethodV2Accepted) {
+ return !"HTTP".equalsIgnoreCase(method) || proxyMethodV2Accepted;
+ }
+
private Response send(String method, String path, JsonObject body) throws Exception {
HttpRequest request = HttpRequest.newBuilder(settings.endpoint().resolve(path))
.timeout(Duration.ofMillis(settings.requestTimeoutMillis())).header("Content-Type", "application/json")
@@ -880,10 +1182,15 @@ private JsonObject sessionBody() {
return body;
}
- static void addCapabilities(JsonObject body, boolean rewardFilesSupported) {
+ static void addCapabilities(JsonObject body) {
+ addCapabilities(body, false, false);
+ }
+
+ static void addCapabilities(JsonObject body, boolean rewardFilesSupported, boolean deploymentReady) {
JsonArray capabilities = new JsonArray();
CAPABILITIES.stream().sorted().filter(capability -> rewardFilesSupported
|| !"config.reward-files.v1".equals(capability)).forEach(capabilities::add);
+ if (deploymentReady) capabilities.add(PluginDeploymentService.CAPABILITY);
body.add("capabilities", capabilities);
JsonArray required = new JsonArray();
required.add("config.files.v1");
@@ -961,6 +1268,9 @@ public void close() {
if (polling != null) polling.cancel(false);
ScheduledFuture inspection = inspectionPolling;
if (inspection != null) inspection.cancel(false);
+ ScheduledFuture deployment = deploymentPolling;
+ if (deployment != null) deployment.cancel(false);
+ if (deployments != null) deployments.cancel();
inspectionExecutor.shutdownNow();
if (reload != null && Bukkit.isPrimaryThread()) reload.cancel(false);
awaitShutdown(executor, operation);
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java
new file mode 100644
index 000000000..2b5256738
--- /dev/null
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java
@@ -0,0 +1,494 @@
+package com.bencodez.votingplugin.control;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.charset.StandardCharsets;
+import java.nio.charset.CodingErrorAction;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.util.HexFormat;
+import java.util.Locale;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BooleanSupplier;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+
+import com.bencodez.votingplugin.util.DurableFiles;
+
+/**
+ * Bounded, pull-only staging for the optional {@code plugin.deploy.v1} capability.
+ * It deliberately never reloads a plugin or starts a server. The small on-disk
+ * marker makes a leased task idempotent after a lost result acknowledgement.
+ */
+public final class PluginDeploymentService {
+ public static final String CAPABILITY = "plugin.deploy.v1";
+ public static final long MAX_ARTIFACT_BYTES = 64L * 1024L * 1024L;
+ private static final int BUFFER_BYTES = 32 * 1024;
+ private static final int MAX_JAR_ENTRIES = 10_000;
+ private static final long MAX_INSPECTED_UNCOMPRESSED_BYTES = 256L * 1024L * 1024L;
+ private static final int MAX_PLUGIN_YML_BYTES = 64 * 1024;
+ private static final String MARKER = ".control-deployment";
+ private static final ScheduledExecutorService BODY_DEADLINE_EXECUTOR = Executors.newSingleThreadScheduledExecutor(r -> {
+ Thread thread = new Thread(r, "VotingPlugin-control-artifact-deadline");
+ thread.setDaemon(true);
+ return thread;
+ });
+
+ private final Path target;
+ private final Path root;
+ private final Path marker;
+ private final Path installedBackendJar;
+ private final boolean replaceExisting;
+ private final AtomicBoolean staging = new AtomicBoolean();
+ private final AtomicReference activeResponse = new AtomicReference<>();
+
+ private PluginDeploymentService(Path target, Path installedBackendJar, boolean replaceExisting) throws IOException {
+ this.target = target.toAbsolutePath().normalize();
+ this.installedBackendJar = installedBackendJar == null ? null : installedBackendJar.toAbsolutePath().normalize();
+ this.replaceExisting = replaceExisting;
+ Path parent = this.target.getParent();
+ if (parent == null) throw new IOException("deployment target has no parent");
+ Files.createDirectories(parent);
+ this.root = parent.toRealPath(LinkOption.NOFOLLOW_LINKS);
+ if (!Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(root)) {
+ throw new IOException("deployment staging directory is unsafe");
+ }
+ if (!this.target.startsWith(root) || Files.isSymbolicLink(this.target)) {
+ throw new IOException("deployment target escapes staging directory");
+ }
+ if (Files.exists(this.target, LinkOption.NOFOLLOW_LINKS)
+ && !Files.isRegularFile(this.target, LinkOption.NOFOLLOW_LINKS)) {
+ throw new IOException("deployment target is not a regular file");
+ }
+ this.marker = root.resolve(this.target.getFileName() + MARKER);
+ }
+
+ /** Bukkit consumes an update only when its file name matches the currently loaded plugin JAR. */
+ public static PluginDeploymentService backend(Path updateDirectory, Path currentPluginJar) throws IOException {
+ if (updateDirectory == null) throw new IOException("Bukkit update folder is unavailable");
+ if (currentPluginJar == null || currentPluginJar.getFileName() == null
+ || !currentPluginJar.getFileName().toString().toLowerCase(Locale.ROOT).endsWith(".jar")) {
+ throw new IOException("Bukkit plugin JAR is unavailable");
+ }
+ Path target = updateDirectory.resolve(currentPluginJar.getFileName().toString()).toAbsolutePath().normalize();
+ Path installed = currentPluginJar.toAbsolutePath().normalize();
+ // An empty/disabled Bukkit update folder can resolve to the loaded JAR's
+ // directory. Staging there would overwrite a file that Bukkit currently has open.
+ if (target.equals(installed) || Files.exists(target, LinkOption.NOFOLLOW_LINKS)
+ && Files.exists(installed, LinkOption.NOFOLLOW_LINKS) && Files.isSameFile(target, installed)) {
+ throw new IOException("Bukkit update folder resolves to the loaded plugin JAR");
+ }
+ return new PluginDeploymentService(target, currentPluginJar, false);
+ }
+
+ /** Proxies atomically replace their discovered plugin JAR only after a durable backup. */
+ public static PluginDeploymentService proxy(Path currentPluginJar) throws IOException {
+ if (currentPluginJar == null || !currentPluginJar.getFileName().toString().endsWith(".jar")) {
+ throw new IOException("proxy plugin JAR is unavailable");
+ }
+ if (!Files.isRegularFile(currentPluginJar, LinkOption.NOFOLLOW_LINKS)
+ || Files.isSymbolicLink(currentPluginJar)) throw new IOException("proxy plugin JAR is unsafe");
+ return new PluginDeploymentService(currentPluginJar, null, true);
+ }
+
+ public boolean isStaging() { return staging.get(); }
+
+ /** Unblocks an in-progress response read during connector shutdown. */
+ public void cancel() {
+ InputStream response = activeResponse.getAndSet(null);
+ if (response != null) {
+ try { response.close(); } catch (IOException ignored) { /* Shutdown is already in progress. */ }
+ }
+ }
+
+ public Result deploy(Task task, URI endpoint, boolean directLocalHosted, String nodeId, UUID sessionId,
+ String credential, HttpClient http, Duration timeout, BooleanSupplier active) {
+ if (!staging.compareAndSet(false, true)) return Result.failure("DEPLOYMENT_FAILED", "Another deployment is still staging");
+ try {
+ validate(task);
+ if (!credentialEndpointAllowed(endpoint, directLocalHosted)) {
+ return Result.failure("INSECURE_ENDPOINT",
+ "Verified update staging requires HTTPS unless Control is hosted directly on this node");
+ }
+ if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before download");
+ if (alreadyStaged(task)) return Result.restartRequired();
+ URI artifact = endpoint.resolve("/api/v1/nodes/" + nodeId + "/deployments/" + task.deploymentId()
+ + "/artifact");
+ HttpRequest request = HttpRequest.newBuilder(artifact).timeout(timeout)
+ .header("Authorization", "Bearer " + credential)
+ .header("X-Node-Session", sessionId.toString())
+ .header("X-Deployment-Attempt", task.attemptId().toString()).GET().build();
+ HttpResponse response = http.send(request, HttpResponse.BodyHandlers.ofInputStream());
+ InputStream body = response.body();
+ activeResponse.set(body);
+ ScheduledFuture bodyDeadline = BODY_DEADLINE_EXECUTOR.schedule(() -> {
+ try { body.close(); } catch (IOException ignored) { /* The staging operation reports the timeout. */ }
+ }, timeout.toMillis(), TimeUnit.MILLISECONDS);
+ try {
+ if (response.statusCode() != 200) {
+ body.close();
+ return Result.failure("DOWNLOAD_FAILED", "Artifact download was rejected");
+ }
+ String contentLength = response.headers().firstValue("Content-Length").orElse(null);
+ if (contentLength != null && (!contentLength.matches("[0-9]{1,9}")
+ || Long.parseLong(contentLength) != task.size())) {
+ body.close();
+ return Result.failure("SIZE_MISMATCH", "Artifact size did not match the deployment task");
+ }
+ if (!active.getAsBoolean()) {
+ body.close();
+ return Result.failure("CANCELLED", "Deployment was cancelled before download");
+ }
+ try (body) {
+ return stage(task, body, active);
+ } catch (IOException failure) {
+ if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before staging");
+ return Result.failure("STAGING_FAILED", "Artifact could not be verified or staged on this node");
+ }
+ } finally {
+ bodyDeadline.cancel(false);
+ activeResponse.compareAndSet(body, null);
+ }
+ } catch (InterruptedException failure) {
+ Thread.currentThread().interrupt();
+ return Result.failure("CANCELLED", "Deployment download was interrupted");
+ } catch (IOException | RuntimeException failure) {
+ return Result.failure("DOWNLOAD_FAILED", "Artifact download failed");
+ } finally {
+ staging.set(false);
+ }
+ }
+
+ /** Package-visible for deterministic artifact-validation tests. */
+ Result stage(Task task, InputStream body, BooleanSupplier active) throws IOException {
+ validate(task);
+ if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before staging");
+ if (alreadyStaged(task)) return Result.restartRequired();
+ activeResponse.compareAndSet(null, body);
+ Path temporary = Files.createTempFile(root, target.getFileName().toString() + ".", ".download");
+ Activation activation = null;
+ try {
+ MessageDigest digest = sha256();
+ long written = copyExact(body, temporary, task.size(), digest, active);
+ force(temporary);
+ if (written != task.size()) return Result.failure("SIZE_MISMATCH", "Artifact size did not match the deployment task");
+ if (!HexFormat.of().formatHex(digest.digest()).equals(task.sha256())) {
+ return Result.failure("HASH_MISMATCH", "Artifact digest did not match the deployment task");
+ }
+ inspectJar(temporary);
+ if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before staging");
+ activation = new Activation();
+ prepareProxyBackup();
+ writeMarker(task, activation);
+ activate(temporary, activation);
+ activation.discard();
+ return Result.restartRequired();
+ } catch (CancelledDeploymentException failure) {
+ return Result.failure("CANCELLED", "Deployment was cancelled before staging");
+ } catch (InvalidArtifactException failure) {
+ return Result.failure("INVALID_ARTIFACT", "Artifact is not a bounded VotingPlugin JAR");
+ } catch (IOException failure) {
+ if (activation != null) {
+ try { activation.rollback(); }
+ catch (IOException rollbackFailure) { failure.addSuppressed(rollbackFailure); }
+ }
+ if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before staging");
+ throw failure;
+ } finally {
+ activeResponse.compareAndSet(body, null);
+ Files.deleteIfExists(temporary);
+ }
+ }
+
+ private long copyExact(InputStream input, Path temporary, long expected, MessageDigest digest,
+ BooleanSupplier active) throws IOException {
+ long total = 0;
+ byte[] buffer = new byte[BUFFER_BYTES];
+ try (var output = Files.newOutputStream(temporary, StandardOpenOption.TRUNCATE_EXISTING)) {
+ for (;;) {
+ if (!active.getAsBoolean()) throw new CancelledDeploymentException();
+ int read = input.read(buffer);
+ if (read == -1) break;
+ total += read;
+ if (total > expected || total > MAX_ARTIFACT_BYTES) throw new InvalidArtifactException();
+ digest.update(buffer, 0, read);
+ output.write(buffer, 0, read);
+ }
+ }
+ return total;
+ }
+
+ private void inspectJar(Path artifact) throws IOException {
+ try (JarFile jar = new JarFile(artifact.toFile(), false)) {
+ int entries = 0;
+ long uncompressed = 0;
+ JarEntry pluginYml = null;
+ var iterator = jar.entries();
+ while (iterator.hasMoreElements()) {
+ JarEntry entry = iterator.nextElement();
+ if (++entries > MAX_JAR_ENTRIES || unsafeEntry(entry.getName())) throw new InvalidArtifactException();
+ long size = entry.getSize();
+ long compressed = entry.getCompressedSize();
+ if (size < 0 || compressed < 0 || compressed > 0 && size > compressed * 200L
+ || (uncompressed += size) > MAX_INSPECTED_UNCOMPRESSED_BYTES) {
+ throw new InvalidArtifactException();
+ }
+ if ("plugin.yml".equals(entry.getName())) {
+ if (pluginYml != null || size > MAX_PLUGIN_YML_BYTES) throw new InvalidArtifactException();
+ pluginYml = entry;
+ }
+ }
+ if (pluginYml == null || !isVotingPluginYml(jar, pluginYml)) throw new InvalidArtifactException();
+ } catch (java.util.zip.ZipException failure) {
+ throw new InvalidArtifactException();
+ }
+ }
+
+ private static boolean unsafeEntry(String name) {
+ if (name == null || name.isEmpty() || name.length() > 512 || name.startsWith("/")
+ || name.startsWith("\\") || name.contains("\\") || name.indexOf('\0') >= 0) return true;
+ for (String component : name.split("/", -1)) {
+ if (component.equals(".") || component.equals("..")) return true;
+ }
+ return false;
+ }
+
+ private static boolean isVotingPluginYml(JarFile jar, JarEntry entry) throws IOException {
+ byte[] bytes;
+ try (InputStream input = jar.getInputStream(entry)) { bytes = input.readNBytes(MAX_PLUGIN_YML_BYTES + 1); }
+ if (bytes.length > MAX_PLUGIN_YML_BYTES) return false;
+ String yml;
+ try {
+ yml = StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT)
+ .onUnmappableCharacter(CodingErrorAction.REPORT).decode(java.nio.ByteBuffer.wrap(bytes)).toString();
+ } catch (java.nio.charset.CharacterCodingException failure) {
+ return false;
+ }
+ boolean found = false;
+ for (String line : yml.split("\\R")) {
+ if (!line.startsWith("name:")) continue;
+ if (found) return false;
+ found = true;
+ String value = line.substring(5).trim();
+ int comment = value.indexOf('#');
+ if (comment >= 0) value = value.substring(0, comment).trim();
+ if (!"VotingPlugin".equals(unquote(value))) return false;
+ }
+ return found;
+ }
+
+ private static String unquote(String value) {
+ return value.length() >= 2 && ((value.startsWith("\"") && value.endsWith("\""))
+ || (value.startsWith("'") && value.endsWith("'"))) ? value.substring(1, value.length() - 1) : value;
+ }
+
+ private void prepareProxyBackup() throws IOException {
+ if (replaceExisting) {
+ if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target)) {
+ throw new IOException("current proxy plugin JAR is unsafe");
+ }
+ Path backup = root.resolve(target.getFileName() + ".control-backup");
+ if (Files.exists(backup, LinkOption.NOFOLLOW_LINKS)
+ && (!Files.isRegularFile(backup, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(backup))) {
+ throw new IOException("proxy plugin backup is unsafe");
+ }
+ Path backupTemp = Files.createTempFile(root, target.getFileName().toString() + ".", ".backup");
+ try {
+ Files.copy(target, backupTemp, StandardCopyOption.REPLACE_EXISTING);
+ force(backupTemp);
+ move(backupTemp, backup);
+ forceDirectory(root);
+ } finally { Files.deleteIfExists(backupTemp); }
+ }
+ }
+
+ private void activate(Path temporary, Activation activation) throws IOException {
+ move(temporary, target);
+ activation.published = true;
+ force(target);
+ forceDirectory(root);
+ }
+
+ /** Holds a private copy until the durable idempotency marker has been published. */
+ private final class Activation {
+ private final Path previous;
+ private boolean published;
+ private boolean markerPublished;
+
+ private Activation() throws IOException {
+ if (!Files.exists(target, LinkOption.NOFOLLOW_LINKS)) {
+ previous = null;
+ return;
+ }
+ if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target)) {
+ throw new IOException("deployment target is unsafe");
+ }
+ previous = Files.createTempFile(root, target.getFileName().toString() + ".", ".rollback");
+ Files.copy(target, previous, StandardCopyOption.REPLACE_EXISTING);
+ force(previous);
+ }
+
+ private void rollback() throws IOException {
+ if (markerPublished) DurableFiles.deleteIfExists(marker);
+ if (!published) {
+ discard();
+ return;
+ }
+ if (previous == null) {
+ if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target)) {
+ throw new IOException("deployed target cannot be safely removed");
+ }
+ Files.delete(target);
+ } else {
+ move(previous, target);
+ force(target);
+ }
+ forceDirectory(root);
+ discard();
+ }
+
+ private void discard() {
+ if (previous == null) return;
+ try { Files.deleteIfExists(previous); }
+ catch (IOException ignored) { /* A private stale rollback copy is safer than a false deployment result. */ }
+ }
+ }
+
+ private void writeMarker(Task task) throws IOException {
+ writeMarker(task, null);
+ }
+
+ private void writeMarker(Task task, Activation activation) throws IOException {
+ Path temporary = Files.createTempFile(root, target.getFileName().toString() + ".", ".marker");
+ try {
+ Files.writeString(temporary, task.deploymentId() + "\n" + task.sha256() + "\n" + task.size() + "\n",
+ StandardCharsets.US_ASCII, StandardOpenOption.TRUNCATE_EXISTING);
+ force(temporary);
+ move(temporary, marker);
+ if (activation != null) activation.markerPublished = true;
+ forceDirectory(root);
+ } finally { Files.deleteIfExists(temporary); }
+ }
+
+ private boolean alreadyStaged(Task task) throws IOException {
+ if (!Files.isRegularFile(marker, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(marker)
+ || Files.size(marker) > 256) return false;
+ if (replaceExisting) {
+ Path backup = root.resolve(target.getFileName() + ".control-backup");
+ if (!Files.isRegularFile(backup, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(backup)) return false;
+ }
+ String[] fields = Files.readString(marker, StandardCharsets.US_ASCII).split("\\R", -1);
+ // A lost acknowledgement can cause Control to issue a new deployment ID
+ // for the same verified artifact after reconnect or restart. The marker
+ // only proves a completed prior stage; the target is checked below by
+ // exact size, digest and plugin identity before acknowledging the retry.
+ if (fields.length < 3 || !task.sha256().equals(fields[1])
+ || !Long.toString(task.size()).equals(fields[2])) return false;
+ if (!replaceExisting && !Files.exists(target, LinkOption.NOFOLLOW_LINKS)) {
+ // Bukkit removes the staged update JAR after consuming it on restart. A
+ // missing staging file alone is not proof: it may have been deleted or
+ // quarantined. Confirm the installed backend JAR is the exact artifact.
+ return installedBackendJar != null && fileMatches(installedBackendJar, task);
+ }
+ return targetMatches(task);
+ }
+
+ private boolean targetMatches(Task task) throws IOException {
+ return fileMatches(target, task);
+ }
+
+ private boolean fileMatches(Path candidate, Task task) throws IOException {
+ if (!Files.isRegularFile(candidate, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(candidate)
+ || Files.size(candidate) != task.size()) return false;
+ MessageDigest digest = sha256();
+ try (InputStream input = Files.newInputStream(candidate, StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS)) {
+ byte[] bytes = new byte[BUFFER_BYTES];
+ for (int read; (read = input.read(bytes)) != -1;) digest.update(bytes, 0, read);
+ }
+ if (!task.sha256().equals(HexFormat.of().formatHex(digest.digest()))) return false;
+ try {
+ inspectJar(candidate);
+ return true;
+ } catch (InvalidArtifactException invalid) {
+ return false;
+ }
+ }
+
+ static void move(Path source, Path destination) throws IOException {
+ // The proxy target is executable on its next startup. A copy/delete fallback
+ // can leave it missing or partial after a crash, even with a durable backup.
+ Files.move(source, destination, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+ }
+
+ private static void force(Path file) throws IOException {
+ try (FileChannel channel = FileChannel.open(file, StandardOpenOption.WRITE)) { channel.force(true); }
+ }
+
+ private static void forceDirectory(Path directory) throws IOException {
+ DurableFiles.forceDirectory(directory);
+ }
+
+ /** True when a deployment bearer credential may be sent to this Control endpoint. */
+ public static boolean credentialEndpointAllowed(URI endpoint, boolean directLocalHosted) {
+ if (endpoint == null) return false;
+ if ("https".equalsIgnoreCase(endpoint.getScheme())) return true;
+ return directLocalHosted && "http".equalsIgnoreCase(endpoint.getScheme())
+ && isLoopbackHost(endpoint.getHost());
+ }
+
+ private static boolean isLoopbackHost(String host) {
+ if (host == null) return false;
+ String normalized = host;
+ if (normalized.length() >= 2 && normalized.charAt(0) == '['
+ && normalized.charAt(normalized.length() - 1) == ']') {
+ normalized = normalized.substring(1, normalized.length() - 1);
+ }
+ if ("localhost".equalsIgnoreCase(normalized) || "::1".equalsIgnoreCase(normalized)
+ || "0:0:0:0:0:0:0:1".equalsIgnoreCase(normalized)) return true;
+ String[] octets = normalized.split("\\.", -1);
+ if (octets.length != 4 || !"127".equals(octets[0])) return false;
+ for (int index = 1; index < octets.length; index++) {
+ if (!octets[index].matches("[0-9]{1,3}") || Integer.parseInt(octets[index]) > 255) return false;
+ }
+ return true;
+ }
+
+ private static MessageDigest sha256() {
+ try { return MessageDigest.getInstance("SHA-256"); }
+ catch (java.security.NoSuchAlgorithmException failure) { throw new IllegalStateException("SHA-256 is unavailable", failure); }
+ }
+
+ private static void validate(Task task) {
+ if (task == null || task.deploymentId() == null || task.attemptId() == null || task.artifactId() == null
+ || !task.artifactId().matches("[A-Za-z0-9][A-Za-z0-9._-]{0,127}") || task.sha256() == null
+ || !task.sha256().matches("[0-9a-fA-F]{64}") || task.size() < 1 || task.size() > MAX_ARTIFACT_BYTES) {
+ throw new IllegalArgumentException("deployment task is invalid");
+ }
+ }
+
+ public record Task(UUID deploymentId, String artifactId, String sha256, long size, UUID attemptId) {
+ public Task { if (sha256 != null) sha256 = sha256.toLowerCase(Locale.ROOT); }
+ }
+ public record Result(boolean success, String code, String message) {
+ static Result restartRequired() { return new Result(true, "RESTART_REQUIRED", "Plugin update staged; restart is required"); }
+ static Result failure(String code, String message) { return new Result(false, code, message); }
+ }
+ private static final class InvalidArtifactException extends IOException { private static final long serialVersionUID = 1L; }
+ private static final class CancelledDeploymentException extends IOException { private static final long serialVersionUID = 1L; }
+}
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/listeners/VotiferEvent.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/listeners/VotiferEvent.java
index 31979e197..5e7188a5f 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/listeners/VotiferEvent.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/listeners/VotiferEvent.java
@@ -41,10 +41,11 @@ public void processVote(String voteSite, String voteUsername) {
if (plugin.getBungeeSettings().isUseBungeecoord() && !plugin.getBungeeSettings().isVotifierBypass()
&& (plugin.getBackendProxyHandler().getMethod().equals(BungeeMethod.PLUGINMESSAGING)
|| plugin.getBackendProxyHandler().getMethod().equals(BungeeMethod.SOCKETS)
+ || plugin.getBackendProxyHandler().getMethod().equals(BungeeMethod.HTTP)
|| plugin.getBackendProxyHandler().getMethod().equals(BungeeMethod.MQTT)
|| plugin.getBackendProxyHandler().getMethod().equals(BungeeMethod.REDIS))) {
plugin.getLogger().severe(
- "Ignoring vote from votifier since pluginmessaging, socket, redis, or mqtt bungee method is enabled, this means you aren't setup correctly for those methods, please check: https://github.com/BenCodez/VotingPlugin/wiki/Bungeecord-Setups");
+ "Ignoring vote from votifier since a proxy vote transport is enabled; receive votes on the proxy or enable VotifierBypass, then check: https://github.com/BenCodez/VotingPlugin/wiki/Bungeecord-Setups");
return;
}
@@ -123,7 +124,7 @@ public void onVotiferEvent(VotifierEvent event) {
if (!MinecraftUsernameValidator.isValid(voteUsername, plugin.getOptions().getBedrockPlayerPrefix())) {
plugin.getLogger().warning("Rejected vote with invalid Minecraft username '"
+ MinecraftUsernameValidator.sanitizeForLog(voteUsername) + "' from service '"
- + MinecraftUsernameValidator.sanitizeForLog(voteSite) + "'");
+ + ServiceSiteValidator.sanitizeForLog(voteSite) + "'");
return;
}
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/BungeeMethod.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/BungeeMethod.java
index c2cb5d9c8..2142037e8 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/BungeeMethod.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/BungeeMethod.java
@@ -8,9 +8,11 @@ public enum BungeeMethod {
MYSQL,
/** Plugin messaging channel. */
PLUGINMESSAGING,
- /** Socket connection. */
- SOCKETS,
- /** Redis connection. */
+ /** Socket connection. */
+ SOCKETS,
+ /** Encrypted single-port HTTP connector. */
+ HTTP,
+ /** Redis connection. */
REDIS,
/** MQTT message broker. */
MQTT;
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java
index 329681705..cb662eba2 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java
@@ -1,9 +1,14 @@
package com.bencodez.votingplugin.proxy;
-import java.util.Collections;
-import java.util.LinkedHashSet;
-import java.util.Set;
-import java.util.UUID;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Locale;
import com.bencodez.votingplugin.timequeue.VoteTimeQueue;
@@ -46,6 +51,26 @@ public class OfflineBungeeVote {
@Getter
@Setter
private boolean deliveryStateDirty;
+ /** SQL primary key for this vote's server-keyed cache row, when applicable. */
+ @Getter
+ @Setter
+ private int serverVoteCacheRowId;
+ /** JSON entry key for this vote's server-keyed emergency cache row, when applicable. */
+ @Getter
+ @Setter
+ private String serverVoteCacheJsonKey;
+ /** SQL primary key for this vote's voter-keyed cache row, when applicable. */
+ @Getter
+ @Setter
+ private int onlineVoteCacheRowId;
+ /** JSON entry key for this vote's voter-keyed emergency cache row, when applicable. */
+ @Getter
+ @Setter
+ private String onlineVoteCacheJsonKey;
+ /** Stable HTTP delivery IDs that must be reused for each target server. */
+ private final Map httpDeliveryIds;
+ /** Stable HTTP standalone-broadcast IDs, separate from reward delivery IDs. */
+ private final Map httpBroadcastDeliveryIds;
/**
* Constructor with UUID voteId.
@@ -94,10 +119,29 @@ public OfflineBungeeVote(UUID voteId, String playerName, String uuid, String ser
* @param broadcastForwardedServers targets that accepted standalone delivery
* @param rewardDelivered whether the cached reward vote was already delivered
*/
- public OfflineBungeeVote(UUID voteId, String playerName, String uuid, String service, long time, boolean realVote,
- String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, Set broadcastTargets,
- Set broadcastForwardedServers, boolean rewardDelivered) {
- this.playerName = playerName;
+ public OfflineBungeeVote(UUID voteId, String playerName, String uuid, String service, long time, boolean realVote,
+ String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, Set broadcastTargets,
+ Set broadcastForwardedServers, boolean rewardDelivered) {
+ this(voteId, playerName, uuid, service, time, realVote, text, broadcastForwarded, proxyBroadcastHandled,
+ broadcastTargets, broadcastForwardedServers, rewardDelivered, Collections.emptyMap());
+ }
+
+ /**
+ * Constructor with full proxy state and stable HTTP delivery IDs.
+ * @param httpDeliveryIds stable HTTP delivery ID by target server
+ */
+ public OfflineBungeeVote(UUID voteId, String playerName, String uuid, String service, long time, boolean realVote,
+ String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, Set broadcastTargets,
+ Set broadcastForwardedServers, boolean rewardDelivered, Map httpDeliveryIds) {
+ this(voteId, playerName, uuid, service, time, realVote, text, broadcastForwarded, proxyBroadcastHandled,
+ broadcastTargets, broadcastForwardedServers, rewardDelivered, httpDeliveryIds, Collections.emptyMap());
+ }
+
+ public OfflineBungeeVote(UUID voteId, String playerName, String uuid, String service, long time, boolean realVote,
+ String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, Set broadcastTargets,
+ Set broadcastForwardedServers, boolean rewardDelivered, Map httpDeliveryIds,
+ Map httpBroadcastDeliveryIds) {
+ this.playerName = playerName;
this.uuid = uuid;
this.service = service;
this.time = time;
@@ -106,10 +150,16 @@ public OfflineBungeeVote(UUID voteId, String playerName, String uuid, String ser
this.voteId = voteId;
this.broadcastForwarded = broadcastForwarded;
this.proxyBroadcastHandled = proxyBroadcastHandled;
- setBroadcastTargets(broadcastTargets);
- setBroadcastForwardedServers(broadcastForwardedServers);
- this.rewardDelivered = rewardDelivered;
- }
+ setBroadcastTargets(broadcastTargets);
+ setBroadcastForwardedServers(broadcastForwardedServers);
+ this.rewardDelivered = rewardDelivered;
+ this.httpDeliveryIds = new LinkedHashMap<>();
+ if (httpDeliveryIds != null) {
+ httpDeliveryIds.forEach(this::setHttpDeliveryId);
+ }
+ this.httpBroadcastDeliveryIds = new LinkedHashMap<>();
+ if (httpBroadcastDeliveryIds != null) httpBroadcastDeliveryIds.forEach(this::setHttpBroadcastDeliveryId);
+ }
/**
* Constructor with String voteId.
@@ -157,12 +207,28 @@ public OfflineBungeeVote(String voteId, String playerName, String uuid, String s
* @param broadcastForwardedServers targets that accepted standalone delivery
* @param rewardDelivered whether the cached reward vote was already delivered
*/
- public OfflineBungeeVote(String voteId, String playerName, String uuid, String service, long time, boolean realVote,
- String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, Set broadcastTargets,
- Set broadcastForwardedServers, boolean rewardDelivered) {
- this(parseVoteId(voteId), playerName, uuid, service, time, realVote, text, broadcastForwarded,
- proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, rewardDelivered);
- }
+ public OfflineBungeeVote(String voteId, String playerName, String uuid, String service, long time, boolean realVote,
+ String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, Set broadcastTargets,
+ Set broadcastForwardedServers, boolean rewardDelivered) {
+ this(parseVoteId(voteId), playerName, uuid, service, time, realVote, text, broadcastForwarded,
+ proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, rewardDelivered);
+ }
+
+ public OfflineBungeeVote(String voteId, String playerName, String uuid, String service, long time, boolean realVote,
+ String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, Set broadcastTargets,
+ Set broadcastForwardedServers, boolean rewardDelivered, Map httpDeliveryIds) {
+ this(parseVoteId(voteId), playerName, uuid, service, time, realVote, text, broadcastForwarded,
+ proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, rewardDelivered, httpDeliveryIds);
+ }
+
+ public OfflineBungeeVote(String voteId, String playerName, String uuid, String service, long time, boolean realVote,
+ String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, Set broadcastTargets,
+ Set broadcastForwardedServers, boolean rewardDelivered, Map httpDeliveryIds,
+ Map httpBroadcastDeliveryIds) {
+ this(parseVoteId(voteId), playerName, uuid, service, time, realVote, text, broadcastForwarded,
+ proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, rewardDelivered, httpDeliveryIds,
+ httpBroadcastDeliveryIds);
+ }
private static UUID parseVoteId(String voteId) {
return voteId == null || voteId.isEmpty() ? null : UUID.fromString(voteId);
@@ -198,12 +264,118 @@ public boolean isProxyBroadcastComplete() {
* @param server backend server receiving the cached vote
* @return true when this server is an original undelivered target
*/
- public boolean needsBroadcastOn(String server) {
+ public boolean needsBroadcastOn(String server) {
if (!proxyBroadcastHandled) {
return !broadcastForwarded;
}
return server != null && broadcastTargets.contains(server) && !broadcastForwardedServers.contains(server);
- }
+ }
+
+ /**
+ * Returns the stable HTTP delivery ID for a target, if one is pending.
+ * @param server target server
+ * @return stable delivery ID or null
+ */
+ public String getHttpDeliveryId(String server) {
+ return server == null ? null : httpDeliveryIds.get(server.toLowerCase(Locale.ROOT));
+ }
+
+ /** Returns a copy of stable reward delivery IDs by normalized target key. */
+ public Map getHttpDeliveryIds() {
+ return new LinkedHashMap<>(httpDeliveryIds);
+ }
+
+ /**
+ * Stores or removes a stable HTTP delivery ID for a target.
+ * @param server target server
+ * @param deliveryId stable delivery ID, or null to remove
+ */
+ public void setHttpDeliveryId(String server, String deliveryId) {
+ if (server == null || server.isBlank()) return;
+ String key = server.toLowerCase(Locale.ROOT);
+ if (deliveryId == null || deliveryId.isBlank()) httpDeliveryIds.remove(key);
+ else httpDeliveryIds.put(key, deliveryId);
+ }
+
+ public String getHttpBroadcastDeliveryId(String server) {
+ return server == null ? null : httpBroadcastDeliveryIds.get(server.toLowerCase(Locale.ROOT));
+ }
+
+ public void setHttpBroadcastDeliveryId(String server, String deliveryId) {
+ if (server == null || server.isBlank()) return;
+ String key = server.toLowerCase(Locale.ROOT);
+ if (deliveryId == null || deliveryId.isBlank()) httpBroadcastDeliveryIds.remove(key);
+ else httpBroadcastDeliveryIds.put(key, deliveryId);
+ }
+
+ /**
+ * Returns a copy of pending standalone HTTP delivery IDs for cache handoff.
+ * @return pending standalone delivery IDs by target server
+ */
+ public Map getHttpBroadcastDeliveryIds() {
+ return new LinkedHashMap<>(httpBroadcastDeliveryIds);
+ }
+
+ /** Returns whether any reward or standalone HTTP delivery is still pending. */
+ public boolean hasPendingHttpDeliveryIds() {
+ return !httpDeliveryIds.isEmpty() || !httpBroadcastDeliveryIds.isEmpty();
+ }
+
+ /**
+ * Encodes stable delivery IDs for JSON/SQL cache storage.
+ * @return bounded delimiter-safe encoding
+ */
+ public String encodeHttpDeliveryIds() {
+ return encodeDeliveryIds(httpDeliveryIds);
+ }
+
+ public String encodeHttpBroadcastDeliveryIds() {
+ return encodeDeliveryIds(httpBroadcastDeliveryIds);
+ }
+
+ private static String encodeDeliveryIds(Map values) {
+ StringBuilder encoded = new StringBuilder();
+ for (Map.Entry entry : values.entrySet()) {
+ if (encoded.length() > 0) encoded.append('.');
+ encoded.append(Base64.getUrlEncoder().withoutPadding()
+ .encodeToString(entry.getKey().getBytes(StandardCharsets.UTF_8)));
+ encoded.append('~');
+ encoded.append(Base64.getUrlEncoder().withoutPadding()
+ .encodeToString(entry.getValue().getBytes(StandardCharsets.UTF_8)));
+ }
+ return encoded.toString();
+ }
+
+ /**
+ * Decodes stable delivery IDs from cache storage. Malformed entries are ignored.
+ * @param encoded encoded map
+ * @return decoded map
+ */
+ public static Map decodeHttpDeliveryIds(String encoded) {
+ return decodeDeliveryIds(encoded);
+ }
+
+ public static Map decodeHttpBroadcastDeliveryIds(String encoded) {
+ return decodeDeliveryIds(encoded);
+ }
+
+ private static Map decodeDeliveryIds(String encoded) {
+ Map decoded = new LinkedHashMap<>();
+ if (encoded == null || encoded.isBlank()) return decoded;
+ for (String entry : encoded.split("\\.", -1)) {
+ int separator = entry.indexOf('~');
+ if (separator <= 0 || separator == entry.length() - 1) continue;
+ try {
+ String server = new String(Base64.getUrlDecoder().decode(entry.substring(0, separator)), StandardCharsets.UTF_8);
+ String deliveryId = new String(Base64.getUrlDecoder().decode(entry.substring(separator + 1)), StandardCharsets.UTF_8);
+ if (!server.isBlank() && deliveryId.matches("[0-9a-fA-F-]{36}"))
+ decoded.put(server.toLowerCase(Locale.ROOT), deliveryId);
+ } catch (IllegalArgumentException ignored) {
+ // Ignore corrupt optional delivery state and retain the vote itself.
+ }
+ }
+ return decoded;
+ }
/**
* Encodes the original broadcast targets for cache storage.
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java
index 841ed2eba..be7053e6b 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java
@@ -11,6 +11,11 @@
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
+import java.nio.file.Files;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Base64;
import java.sql.SQLException;
import java.time.Duration;
import java.time.Instant;
@@ -21,9 +26,12 @@
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
+import java.util.Queue;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
@@ -34,6 +42,7 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
import javax.net.ssl.SSLParameters;
@@ -49,6 +58,9 @@
import com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec;
import com.bencodez.simpleapi.servercomm.global.GlobalMessageListener;
import com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler;
+import com.bencodez.simpleapi.servercomm.http.HttpEnrollmentAuthority;
+import com.bencodez.simpleapi.servercomm.http.HttpProxyTransportServer;
+import com.bencodez.simpleapi.servercomm.http.HttpTlsIdentity;
import com.bencodez.simpleapi.servercomm.mqtt.MqttHandler;
import com.bencodez.simpleapi.servercomm.mqtt.MqttServerComm;
import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger;
@@ -66,6 +78,7 @@
import com.bencodez.simpleapi.sql.mysql.config.MysqlConfig;
import com.bencodez.votingplugin.proxy.broadcast.ProxyBroadcastDecider;
import com.bencodez.votingplugin.proxy.cache.IVoteCache;
+import com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects;
import com.bencodez.votingplugin.proxy.cache.VoteCacheHandler;
import com.bencodez.votingplugin.proxy.cache.nonvoted.INonVotedPlayersStorage;
import com.bencodez.votingplugin.proxy.cache.nonvoted.NonVotedPlayersCache;
@@ -79,6 +92,7 @@
import com.bencodez.votingplugin.proxy.presence.PlayerPresence;
import com.bencodez.votingplugin.timequeue.VoteTimeQueue;
import com.bencodez.votingplugin.topvoter.TopVoter;
+import com.bencodez.votingplugin.util.DurableFiles;
import com.bencodez.votingplugin.util.MinecraftUsernameValidator;
import com.bencodez.votingplugin.util.ServiceSiteValidator;
import com.bencodez.votingplugin.votelog.VoteLogMysqlTable;
@@ -95,11 +109,102 @@
import lombok.Setter;
public abstract class VotingPluginProxy {
+ public static final class VoteRetryException extends IllegalStateException {
+ private static final long serialVersionUID = 1L;
+
+ private VoteRetryException() {
+ super("Vote processing could not be made durable; retry is required");
+ }
+ }
private static final long PRESENCE_HANDOFF_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(2);
private static final long PRESENCE_STARTUP_RESYNC_DELAY_SECONDS = 5L;
+ private static final int MAX_LIVE_VOTE_RETRIES = 1024;
+ private static final int MAX_MULTI_PROXY_VOTE_ATTEMPTS = 12;
+ private static final int MAX_MULTI_PROXY_VOTE_RETRIES = 1024;
+ private static final int MAX_COMPLETED_MULTI_PROXY_VOTES = 4096;
+ private static final String FORWARDED_QUEUE_TOTALS_PREFIX = "\u0000VP-FWD:";
+ private static final int FINAL_SHUTDOWN_PERSISTENCE_ATTEMPTS = 3;
+ private static final String REWARD_JOURNAL_TARGET_PREFIX = "__vp_reward_target__:";
+ private final Map liveVoteRetries = new LinkedHashMap<>();
+ private final Map multiProxyVoteRetries = new LinkedHashMap<>();
+ private final LinkedHashMap completedMultiProxyVotes = new LinkedHashMap<>();
+ // Set only after all replacement gates have succeeded. Vote entry points are
+ // synchronized, so no new side-effecting vote can race the handoff window.
+ private boolean runtimeReplacementPrepared;
+
+ private static final class LiveVoteRetryState {
+ private String requestIdentity;
+ private VoteTotalsSnapshot totals;
+ private ArrayList totalsInput;
+ private boolean votePartyApplied;
+ private boolean totalsApplied;
+ private final Set broadcastForwardedServers = new LinkedHashSet<>();
+ private final Set deliveredRewardServers = new LinkedHashSet<>();
+ private final Map rewardStates = new LinkedHashMap<>();
+ private Set rewardServers;
+ private boolean rewardJournalsDurable;
+ private boolean multiProxyForwardingHandled;
+ private OfflineBungeeVote standaloneBroadcastState;
+ private OfflineBungeeVote rewardJournalOwner;
+ private OfflineBungeeVote pendingOnlineRewardState;
+ private VoteTimeQueue queuedVote;
+ private String player;
+ private String service;
+ private String uuid;
+ private long time;
+ private boolean realVote;
+
+ }
+
+ private final class MultiProxyVoteRetry implements Runnable {
+ private enum Phase {
+ EXECUTE,
+ PERSIST_DEFERRED_RECEIPT,
+ PERSIST_COMPLETION
+ }
+
+ private final String player;
+ private final String service;
+ private final boolean realVote;
+ private final boolean timeQueue;
+ private final long queueTime;
+ private final VoteTotalsSnapshot totals;
+ private final String uuid;
+ private final UUID voteId;
+ private final String origin;
+ private int attempts;
+ private boolean scheduled;
+ private Phase phase = Phase.EXECUTE;
+
+ private MultiProxyVoteRetry(String player, String service, boolean realVote, boolean timeQueue, long queueTime,
+ VoteTotalsSnapshot totals, String uuid, UUID voteId, String origin) {
+ this.player = player;
+ this.service = service;
+ this.realVote = realVote;
+ this.timeQueue = timeQueue;
+ this.queueTime = queueTime;
+ this.totals = totals;
+ this.uuid = uuid;
+ this.voteId = voteId;
+ this.origin = origin == null ? "" : origin;
+ }
+
+ @Override
+ public void run() {
+ attemptMultiProxyVote(this);
+ }
+ }
private static final long PRESENCE_MAINTENANCE_INTERVAL_SECONDS = 30L;
private static final long PRESENCE_BACKEND_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(90);
private static final long CONTROL_ENROLLMENT_MIN_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(10);
+ private static final int MAX_PENDING_VOTE_PARTY_REWARDS = 1024;
+ private static final long HTTP_TRANSPORT_RECONCILIATION_DELAY_MILLIS = 100L;
+ // Acks run before SimpleAPI removes an entry. Keep one bounded, single-flight
+ // poll armed while a replacement is deferred so state cleared after an ack (or
+ // from the durable cache) cannot strand the old HTTP runtime indefinitely.
+ private static final long HTTP_TRANSPORT_RECONCILIATION_POLL_MILLIS = 1_000L;
+ private static final String HTTP_RETAINED_LISTENER_MAGIC = "VotingPlugin HTTP retained listener v1";
+ private static final Map PREPARED_HTTP_TRANSPORTS = new ConcurrentHashMap<>();
@Getter
@Setter
@@ -118,6 +223,53 @@ public abstract class VotingPluginProxy {
private HashMap clientHandles;
private SocketHandler socketHandler;
+ private HttpProxyTransportServer httpTransportServer;
+ private HttpEnrollmentAuthority httpEnrollmentAuthority;
+ private String liveHttpHost;
+ private String liveHttpPublicEndpoint;
+ private int liveHttpPort;
+ private HttpListenerSettings retainedHttpStartupSettings;
+
+ private static final class HttpListenerSettings {
+ private final String host;
+ private final int port;
+ private final String publicEndpoint;
+
+ private HttpListenerSettings(String host, int port, String publicEndpoint) {
+ this.host = host;
+ this.port = port;
+ this.publicEndpoint = publicEndpoint;
+ }
+ }
+
+ private static final class PreparedHttpTransport {
+ private final HttpProxyTransportServer server;
+ private final HttpEnrollmentAuthority authority;
+ private final AtomicReference owner;
+ private final String host;
+ private final int port;
+ private final String publicEndpoint;
+
+ private PreparedHttpTransport(HttpProxyTransportServer server, HttpEnrollmentAuthority authority,
+ AtomicReference owner, String host, int port, String publicEndpoint) {
+ this.server = server;
+ this.authority = authority;
+ this.owner = owner;
+ this.host = host;
+ this.port = port;
+ this.publicEndpoint = publicEndpoint;
+ }
+
+ private boolean matches(VotingPluginProxyConfig config) {
+ return java.util.Objects.equals(host, config.getHttpHost()) && port == config.getHttpPort()
+ && java.util.Objects.equals(publicEndpoint, config.getHttpPublicEndpoint());
+ }
+
+ private void close() {
+ owner.set(null);
+ server.close();
+ }
+ }
@Getter
@Setter
@@ -137,6 +289,15 @@ public abstract class VotingPluginProxy {
private boolean timeVoteRetryScheduled;
private boolean timeVoteDeliveryRetryScheduled;
private boolean cachedVoteDeliveryRetryScheduled;
+ private boolean votePartyDeliveryRetryScheduled;
+ private boolean deferredHttpTransportReconciliation;
+ private boolean httpTransportReconciliationScheduled;
+ private boolean httpTransportReconciliationRunning;
+ private long httpTransportReconciliationGeneration;
+ private long votePartyProxyCommandAttemptSequence;
+ private long votePartyProxyCommandInFlight;
+ private volatile CompletableFuture votePartyProxyCommandExecution;
+ private boolean votePartyProxyCommandCompletedUnpersisted;
private boolean enabled;
@@ -524,16 +685,118 @@ public void addVoteParty() {
private Set sendProxyBroadcast(Set targets, String uuid, String player, String service, long time,
String text, boolean wasOnline) {
+ return sendProxyBroadcast(targets, uuid, player, service, time, text, wasOnline,
+ (OfflineBungeeVote) null);
+ }
+
+ private Set sendProxyBroadcast(Set targets, String uuid, String player, String service, long time,
+ String text, boolean wasOnline, OfflineBungeeVote cachedVote) {
+ Set forwarded = new LinkedHashSet<>();
+ for (String targetServer : targets) {
+ JsonEnvelope envelope = VotingPluginWire.voteBroadcast(uuid, player, service, time, text, wasOnline);
+ boolean accepted = method == BungeeMethod.HTTP
+ ? sendHttpBroadcastEnvelopeWithRecovery(targetServer, envelope, cachedVote)
+ : sendProxyBroadcastEnvelopeNow(targetServer, envelope);
+ if (accepted) {
+ forwarded.add(targetServer);
+ }
+ }
+ return forwarded;
+ }
+
+ private Set sendProxyBroadcast(Set targets, String uuid, String player, String service, long time,
+ String text, boolean wasOnline, VoteTimeQueue cachedVote) {
Set forwarded = new LinkedHashSet<>();
for (String targetServer : targets) {
+ // A timed vote can survive a proxy restart. Persist its HTTP delivery ID before
+ // publication so a crash after the transport accepts it replays with the same
+ // ID rather than creating a second backend broadcast.
+ if (method == BungeeMethod.HTTP && !prepareTimedHttpBroadcastDelivery(targetServer, cachedVote)) {
+ continue;
+ }
JsonEnvelope envelope = VotingPluginWire.voteBroadcast(uuid, player, service, time, text, wasOnline);
- if (sendProxyBroadcastEnvelopeNow(targetServer, envelope)) {
+ boolean accepted = method == BungeeMethod.HTTP
+ ? sendHttpBroadcastEnvelopeWithRecovery(targetServer, envelope, cachedVote)
+ : sendProxyBroadcastEnvelopeNow(targetServer, envelope);
+ if (accepted) {
forwarded.add(targetServer);
+ if (method == BungeeMethod.HTTP) {
+ cachedVote.getBroadcastForwardedServers().add(targetServer);
+ // Persist completion before preparing another target. Otherwise persisting
+ // that target's ID could leave this accepted target looking pending after a
+ // crash, and it would be replayed under a newly generated ID.
+ if (!persistTimeVoteDelivery(cachedVote)) break;
+ } else if (cachedVote.getHttpBroadcastDeliveryId(targetServer) != null) {
+ cachedVote.setHttpBroadcastDeliveryId(targetServer, null);
+ cachedVote.setDeliveryStateDirty(true);
+ }
}
}
return forwarded;
}
+ private boolean prepareTimedHttpBroadcastDelivery(String server, VoteTimeQueue vote) {
+ if (vote.getHttpBroadcastDeliveryId(server) != null) return true;
+ vote.setHttpBroadcastDeliveryId(server, UUID.randomUUID().toString());
+ vote.setDeliveryStateDirty(true);
+ return persistTimeVoteDelivery(vote);
+ }
+
+ protected boolean sendHttpBroadcastEnvelopeWithRecovery(String server, JsonEnvelope envelope,
+ OfflineBungeeVote cachedVote) {
+ String persistedId = cachedVote == null ? null : cachedVote.getHttpBroadcastDeliveryId(server);
+ String stableId = persistedId != null ? persistedId
+ : stableCachedHttpDeliveryId("broadcast", server, envelope, cachedVote);
+ try {
+ boolean accepted = stableId == null ? sendProxyBroadcastEnvelopeNow(server, envelope)
+ : sendHttpEnvelope(server, stableId, envelope);
+ if (accepted && persistedId != null) {
+ cachedVote.setHttpBroadcastDeliveryId(server, null);
+ cachedVote.setDeliveryStateDirty(true);
+ }
+ return accepted;
+ } catch (HttpProxyTransportServer.DeliveryRetryException failure) {
+ if (cachedVote != null) {
+ cachedVote.setHttpBroadcastDeliveryId(server, failure.deliveryId());
+ cachedVote.setDeliveryStateDirty(true);
+ // Let the caller persist the recovered ID before retrying. Retrying here
+ // would create a crash window after acceptance but before durable cache state.
+ return false;
+ }
+ try {
+ boolean accepted = sendHttpEnvelope(server, failure.deliveryId(), envelope);
+ return accepted;
+ } catch (RuntimeException retryFailure) {
+ debug("Unable to recover HTTP standalone delivery " + failure.deliveryId() + ": "
+ + retryFailure.getMessage());
+ return false;
+ }
+ } catch (RuntimeException failure) {
+ debug("Unable to send HTTP standalone delivery: " + failure.getMessage());
+ return false;
+ }
+ }
+
+ private boolean sendHttpBroadcastEnvelopeWithRecovery(String server, JsonEnvelope envelope,
+ VoteTimeQueue cachedVote) {
+ String stableId = cachedVote.getHttpBroadcastDeliveryId(server);
+ if (stableId == null) {
+ debug("Skipping HTTP timed broadcast without a persisted delivery ID for " + server);
+ return false;
+ }
+ try {
+ boolean accepted = sendHttpEnvelope(server, stableId, envelope);
+ if (accepted) {
+ cachedVote.setHttpBroadcastDeliveryId(server, null);
+ cachedVote.setDeliveryStateDirty(true);
+ }
+ return accepted;
+ } catch (RuntimeException failure) {
+ debug("Unable to send HTTP timed broadcast delivery: " + failure.getMessage());
+ return false;
+ }
+ }
+
/**
* Sends a standalone proxy broadcast through the selected transport and reports
* whether that transport accepted the message.
@@ -566,11 +829,36 @@ protected boolean sendProxyBroadcastEnvelopeNow(String server, JsonEnvelope enve
// envelopes. This preserves the socket connection and its delivery
// acknowledgement instead of creating a second short-lived socket.
return sendSocketEnvelope(server, envelope);
+ case HTTP:
+ return sendHttpEnvelope(server, envelope);
default:
return false;
}
}
+ /**
+ * Sends a reward-bearing vote envelope and reports whether the selected
+ * transport accepted it. Legacy transports retain their existing asynchronous
+ * semantics; HTTP exposes its bounded-queue result so a vote is never discarded
+ * when the queue is full.
+ */
+ protected boolean sendVoteEnvelopeAccepted(String server, int delay, JsonEnvelope envelope) {
+ return sendVoteEnvelopeAccepted(server, delay, envelope, null);
+ }
+
+ protected boolean sendVoteEnvelopeAccepted(String server, int delay, JsonEnvelope envelope,
+ OfflineBungeeVote cachedVote) {
+ if (method == BungeeMethod.HTTP) {
+ return sendHttpEnvelopeWithRecovery(server, envelope, cachedVote);
+ }
+ GlobalMessageProxyHandler handler = globalMessageProxyHandler;
+ if (handler == null) {
+ return false;
+ }
+ handler.sendMessage(server, delay, envelope);
+ return true;
+ }
+
public synchronized void checkCachedVotes(String server) {
int delay = 1;
if (isServerValid(server)) {
@@ -588,13 +876,18 @@ public synchronized void checkCachedVotes(String server) {
if (cache.isProxyBroadcastHandled() && cache.needsBroadcastOn(server)) {
Set forwarded = sendProxyBroadcast(Collections.singleton(server),
cache.getUuid(), cache.getPlayerName(), cache.getService(), cache.getTime(),
- cache.getText(), false);
- if (cache.getBroadcastForwardedServers().addAll(forwarded)) {
+ cache.getText(), false, cache);
+ boolean broadcastChanged = cache.getBroadcastForwardedServers().addAll(forwarded);
+ if (broadcastChanged) {
cache.setBroadcastForwarded(cache.isProxyBroadcastComplete());
- if (!persistServerVoteDelivery(server, cache)) {
- continue;
- }
}
+ if ((broadcastChanged || cache.isDeliveryStateDirty())
+ && !persistServerVoteDelivery(server, cache)) continue;
+ }
+ if (cache.isRewardDelivered()) {
+ if (cache.isProxyBroadcastHandled() && !cache.isProxyBroadcastComplete()) continue;
+ removed.add(cache);
+ continue;
}
boolean toSend = true;
@@ -607,7 +900,7 @@ public synchronized void checkCachedVotes(String server) {
}
}
if (toSend) {
- boolean broadcastHere = cache.needsBroadcastOn(server);
+ boolean broadcastHere = !cache.isProxyBroadcastHandled() && cache.needsBroadcastOn(server);
if (!cache.isProxyBroadcastHandled() && broadcastHere
&& getConfig().getProxyBroadcastEnabled()) {
boolean playerOnline = isPlayerOnlineForVoteRouting(cache.getPlayerName());
@@ -619,13 +912,24 @@ && getConfig().getProxyBroadcastEnabled()) {
broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets);
}
- globalMessageProxyHandler.sendMessage(server, delay,
+ if (!sendVoteEnvelopeAccepted(server, delay,
VotingPluginWire.vote(cache.getPlayerName(), cache.getUuid(),
cache.getService(), cache.getTime(), false, cache.isRealVote(),
cache.getText(), cache.getVoteId(), getConfig().getBungeeManageTotals(),
- broadcastHere, num, numberOfVotes));
+ broadcastHere, num, numberOfVotes), cache)) {
+ debug("Retaining cached vote because the transport rejected delivery for " + server);
+ persistServerVoteDelivery(server, cache);
+ continue;
+ }
delay++;
num++;
+ cache.setRewardDelivered(true);
+ if (!persistServerVoteDelivery(server, cache)) {
+ continue;
+ }
+ if (cache.isProxyBroadcastHandled() && !cache.isProxyBroadcastComplete()) {
+ continue;
+ }
removed.add(cache);
} else {
debug("Not sending vote because user isn't on server " + server + ": "
@@ -657,8 +961,19 @@ public synchronized void checkOnlineVotes(String player, String uuid, String ser
int num = 1;
int numberOfVotes = (int) c.stream().filter(vote -> !vote.isRewardDelivered()).count();
boolean deliveredReward = false;
- ArrayList retained = new ArrayList<>();
- for (OfflineBungeeVote cache : c) {
+ for (OfflineBungeeVote cache : new ArrayList<>(c)) {
+ if (isIncompleteRewardJournalOwner(cache)) {
+ if (!materializeRewardJournalOwner(uuid, cache)) continue;
+ if (!cache.isProxyBroadcastHandled() || cache.isProxyBroadcastComplete()) {
+ if (!getVoteCacheHandler().tryRemoveOnlineVote(uuid, cache)) {
+ scheduleCachedVoteDeliveryRetry();
+ }
+ }
+ continue;
+ }
+ if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(uuid, cache)) {
+ continue;
+ }
if (cache.isProxyBroadcastHandled()) {
Set pendingTargets = new LinkedHashSet<>(cache.getBroadcastTargets());
pendingTargets.removeAll(cache.getBroadcastForwardedServers());
@@ -666,12 +981,14 @@ public synchronized void checkOnlineVotes(String player, String uuid, String ser
if (blockedServers != null) {
pendingTargets.removeAll(blockedServers);
}
- cache.getBroadcastForwardedServers().addAll(sendProxyBroadcast(pendingTargets,
+ boolean broadcastChanged = cache.getBroadcastForwardedServers().addAll(sendProxyBroadcast(pendingTargets,
cache.getUuid(), cache.getPlayerName(), cache.getService(), cache.getTime(),
- cache.getText(), false));
+ cache.getText(), false, cache));
cache.setBroadcastForwarded(cache.isProxyBroadcastComplete());
+ if ((broadcastChanged || cache.isDeliveryStateDirty())
+ && !persistOnlineVoteDelivery(uuid, cache)) continue;
}
- boolean broadcastHere = cache.needsBroadcastOn(server);
+ boolean broadcastHere = !cache.isProxyBroadcastHandled() && cache.needsBroadcastOn(server);
if (!cache.isProxyBroadcastHandled() && broadcastHere
&& getConfig().getProxyBroadcastEnabled()) {
String playerServer = (server != null) ? server : getCurrentPlayerServerForVoteRouting(player);
@@ -681,31 +998,25 @@ && getConfig().getProxyBroadcastEnabled()) {
}
if (!cache.isRewardDelivered()) {
- globalMessageProxyHandler.sendMessage(server, delay,
+ if (!sendVoteEnvelopeAccepted(server, delay,
VotingPluginWire.voteOnline(cache.getPlayerName(), cache.getUuid(), cache.getService(),
cache.getTime(), false, cache.isRealVote(), cache.getText(), cache.getVoteId(),
- getConfig().getBungeeManageTotals(), broadcastHere, num, numberOfVotes));
- // The normal envelope is also a valid broadcast delivery for the
- // current target. Record it so a previously pending standalone
- // retry cannot announce the same vote again later.
- if (cache.isProxyBroadcastHandled() && broadcastHere) {
- cache.getBroadcastForwardedServers().add(server);
- cache.setBroadcastForwarded(cache.isProxyBroadcastComplete());
+ getConfig().getBungeeManageTotals(), broadcastHere, num, numberOfVotes), cache)) {
+ debug("Retaining online vote because the transport rejected delivery for " + server);
+ persistOnlineVoteDelivery(uuid, cache);
+ continue;
}
cache.setRewardDelivered(true);
+ if (!persistOnlineVoteDelivery(uuid, cache)) continue;
deliveredReward = true;
delay++;
num++;
}
- if (cache.isProxyBroadcastHandled() && !cache.isProxyBroadcastComplete()) {
- retained.add(cache);
+ if (!cache.isProxyBroadcastHandled() || cache.isProxyBroadcastComplete()) {
+ getVoteCacheHandler().removeOnlineVote(uuid, cache);
}
}
- getVoteCacheHandler().removeOnlineVotes(uuid);
- for (OfflineBungeeVote pending : retained) {
- getVoteCacheHandler().addOnlineVote(uuid, pending);
- }
// multiproxy: envelope-only
if (deliveredReward && getConfig().getMultiProxySupport()
@@ -730,6 +1041,12 @@ protected synchronized void retryPendingOnlineBroadcasts(String server) {
}
for (String cachedUuid : getVoteCacheHandler().getOnlineVoteUUIDs()) {
for (OfflineBungeeVote cache : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(cachedUuid))) {
+ if (isIncompleteRewardJournalOwner(cache) && !materializeRewardJournalOwner(cachedUuid, cache)) {
+ continue;
+ }
+ if (retryCompletedRewardJournalOwner(cachedUuid, cache)) {
+ continue;
+ }
if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) {
continue;
}
@@ -737,7 +1054,7 @@ protected synchronized void retryPendingOnlineBroadcasts(String server) {
continue;
}
Set forwarded = sendProxyBroadcast(Collections.singleton(server), cache.getUuid(),
- cache.getPlayerName(), cache.getService(), cache.getTime(), cache.getText(), false);
+ cache.getPlayerName(), cache.getService(), cache.getTime(), cache.getText(), false, cache);
if (cache.getBroadcastForwardedServers().addAll(forwarded)) {
cache.setBroadcastForwarded(cache.isProxyBroadcastComplete());
if (cache.isRewardDelivered() && cache.isProxyBroadcastComplete()) {
@@ -745,6 +1062,8 @@ protected synchronized void retryPendingOnlineBroadcasts(String server) {
} else {
persistOnlineVoteDelivery(cachedUuid, cache);
}
+ } else if (cache.isDeliveryStateDirty()) {
+ persistOnlineVoteDelivery(cachedUuid, cache);
}
}
}
@@ -767,8 +1086,8 @@ protected synchronized void retryPendingTimeBroadcasts(String server) {
continue;
}
Set forwarded = sendProxyBroadcast(Collections.singleton(server), vote.getUuid(), vote.getName(),
- vote.getService(), vote.getTime(), vote.getTotals(), false);
- if (vote.getBroadcastForwardedServers().addAll(forwarded)) {
+ vote.getService(), vote.getTime(), vote.getTotals(), false, vote);
+ if (vote.getBroadcastForwardedServers().addAll(forwarded) || vote.isDeliveryStateDirty()) {
persistTimeVoteDelivery(vote);
}
}
@@ -780,12 +1099,26 @@ protected synchronized void retryPendingTimeBroadcasts(String server) {
* carrier event.
*/
public synchronized void retryPendingOnlineBroadcasts() {
+ if (!getVoteCacheHandler().retryPendingVotePersistence()) {
+ scheduleCachedVoteDeliveryRetry();
+ }
for (String cachedUuid : new LinkedHashSet<>(getVoteCacheHandler().getOnlineVoteUUIDs())) {
for (OfflineBungeeVote cache : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(cachedUuid))) {
+ if (isIncompleteRewardJournalOwner(cache) && !materializeRewardJournalOwner(cachedUuid, cache)) {
+ continue;
+ }
+ if (retryCompletedRewardJournalOwner(cachedUuid, cache)) {
+ continue;
+ }
if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) {
continue;
}
- if (!cache.isProxyBroadcastHandled() || cache.isProxyBroadcastComplete()) {
+ if (cache.isProxyBroadcastHandled() && cache.isRewardDelivered()
+ && cache.isProxyBroadcastComplete()) {
+ getVoteCacheHandler().removeOnlineVote(cachedUuid, cache);
+ continue;
+ }
+ if (!cache.isProxyBroadcastHandled()) {
continue;
}
Set pendingTargets = new LinkedHashSet<>(cache.getBroadcastTargets());
@@ -795,7 +1128,7 @@ public synchronized void retryPendingOnlineBroadcasts() {
pendingTargets.removeAll(blockedServers);
}
Set forwarded = sendProxyBroadcast(pendingTargets, cache.getUuid(), cache.getPlayerName(),
- cache.getService(), cache.getTime(), cache.getText(), false);
+ cache.getService(), cache.getTime(), cache.getText(), false, cache);
if (cache.getBroadcastForwardedServers().addAll(forwarded)) {
cache.setBroadcastForwarded(cache.isProxyBroadcastComplete());
if (cache.isRewardDelivered() && cache.isProxyBroadcastComplete()) {
@@ -803,6 +1136,8 @@ public synchronized void retryPendingOnlineBroadcasts() {
} else {
persistOnlineVoteDelivery(cachedUuid, cache);
}
+ } else if (cache.isDeliveryStateDirty()) {
+ persistOnlineVoteDelivery(cachedUuid, cache);
}
}
}
@@ -827,8 +1162,8 @@ public synchronized void retryPendingTimeBroadcasts() {
pendingTargets.removeAll(blockedServers);
}
Set forwarded = sendProxyBroadcast(pendingTargets, vote.getUuid(), vote.getName(), vote.getService(),
- vote.getTime(), vote.getTotals(), false);
- if (vote.getBroadcastForwardedServers().addAll(forwarded)) {
+ vote.getTime(), vote.getTotals(), false, vote);
+ if (vote.getBroadcastForwardedServers().addAll(forwarded) || vote.isDeliveryStateDirty()) {
persistTimeVoteDelivery(vote);
}
}
@@ -901,6 +1236,9 @@ private void scheduleCachedVoteDeliveryRetry() {
}
private synchronized void retryCachedVoteDeliveryPersistence() {
+ if (!getVoteCacheHandler().retryPendingVotePersistence()) {
+ scheduleCachedVoteDeliveryRetry();
+ }
for (String server : getVoteCacheHandler().getCachedVotesServers()) {
for (OfflineBungeeVote vote : new ArrayList<>(getVoteCacheHandler().getVotes(server))) {
if (vote.isDeliveryStateDirty()) {
@@ -910,6 +1248,7 @@ private synchronized void retryCachedVoteDeliveryPersistence() {
}
for (String uuid : new LinkedHashSet<>(getVoteCacheHandler().getOnlineVoteUUIDs())) {
for (OfflineBungeeVote vote : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(uuid))) {
+ if (retryCompletedRewardJournalOwner(uuid, vote)) continue;
if (vote.isDeliveryStateDirty()) {
persistOnlineVoteDelivery(uuid, vote);
}
@@ -917,36 +1256,107 @@ private synchronized void retryCachedVoteDeliveryPersistence() {
}
}
- public void checkVoteParty() {
- if (getConfig().getVotePartyEnabled()) {
- if (votePartyVotes >= currentVotePartyVotesRequired) {
- debug("Vote party reached");
- addCurrentVotePartyVotes(-currentVotePartyVotesRequired);
+ public synchronized void checkVoteParty() {
+ if (!getConfig().getVotePartyEnabled()) return;
+ if (votePartyVotes < currentVotePartyVotesRequired) {
+ saveVoteCacheFile();
+ return;
+ }
+ if (!retryPendingVotePartyProxyEffects()) {
+ persistRetainedVotePartyThreshold();
+ return;
+ }
+ PendingVotePartyProxyEffects stagedProxyEffects = PendingVotePartyProxyEffects.empty();
+ if (method == BungeeMethod.HTTP) {
+ try {
+ stagedProxyEffects = new PendingVotePartyProxyEffects(getConfig().getVotePartyBroadcast(),
+ getConfig().getVotePartyBungeeCommands());
+ } catch (IllegalArgumentException oversized) {
+ logSevere("HTTP vote-party proxy effects exceed the durable backlog limit; retaining the vote-party threshold");
+ persistRetainedVotePartyThreshold();
+ return;
+ }
+ }
+ Collection targets = getConfig().getVotePartySendToAllServers()
+ ? getAllAvailableServers() : getConfig().getVotePartyServersToSend();
+ Map onlineTargets = onlineVotePartyTargets(targets);
+ if (method == BungeeMethod.HTTP && !canQueueVotePartyRewards(onlineTargets)) {
+ try {
+ saveVotePartyStateDurably();
+ } catch (IOException failure) {
+ throw new IllegalStateException("Unable to retain the full HTTP vote-party backlog", failure);
+ }
+ return;
+ }
- currentVotePartyVotesRequired += getConfig().getVotePartyIncreaseVotesRequired();
- setVoteCacheVotePartyIncreaseVotesRequired(
- getVoteCacheVotePartyIncreaseVotesRequired() + getConfig().getVotePartyIncreaseVotesRequired());
+ Map stagedRewards = new LinkedHashMap<>();
+ if (method == BungeeMethod.HTTP) {
+ for (String canonicalServer : onlineTargets.keySet()) {
+ String deliveryId = UUID.randomUUID().toString();
+ setVoteCachePendingVotePartyReward(canonicalServer, deliveryId, true);
+ stagedRewards.put(canonicalServer, deliveryId);
+ }
+ }
+ int previousVotes = votePartyVotes;
+ int previousRequired = currentVotePartyVotesRequired;
+ int previousIncrease = getVoteCacheVotePartyIncreaseVotesRequired();
+ PendingVotePartyProxyEffects previousProxyEffects = method == BungeeMethod.HTTP
+ ? getVoteCachePendingVotePartyProxyEffects() : PendingVotePartyProxyEffects.empty();
+ if (method == BungeeMethod.HTTP) setVoteCachePendingVotePartyProxyEffects(stagedProxyEffects);
+ debug("Vote party reached");
+ addCurrentVotePartyVotes(-currentVotePartyVotesRequired);
+ currentVotePartyVotesRequired += getConfig().getVotePartyIncreaseVotesRequired();
+ setVoteCacheVotePartyIncreaseVotesRequired(
+ previousIncrease + getConfig().getVotePartyIncreaseVotesRequired());
+ try {
+ if (method == BungeeMethod.HTTP) saveVotePartyStateDurably();
+ else saveVoteCacheFile();
+ } catch (IOException | RuntimeException failure) {
+ votePartyVotes = previousVotes;
+ setVoteCacheVotePartyCurrentVotes(previousVotes);
+ currentVotePartyVotesRequired = previousRequired;
+ setVoteCacheVotePartyIncreaseVotesRequired(previousIncrease);
+ for (Map.Entry staged : stagedRewards.entrySet())
+ setVoteCachePendingVotePartyReward(staged.getKey(), staged.getValue(), false);
+ if (method == BungeeMethod.HTTP) setVoteCachePendingVotePartyProxyEffects(previousProxyEffects);
+ throw failure instanceof RuntimeException runtime ? runtime
+ : new IllegalStateException("Unable to persist HTTP vote-party rewards", failure);
+ }
+
+ if (method == BungeeMethod.HTTP) {
+ if (retryPendingVotePartyProxyEffects()) retryPendingVotePartyRewards();
+ } else {
+ if (!getConfig().getVotePartyBroadcast().isEmpty()) broadcast(getConfig().getVotePartyBroadcast());
+ for (String command : getConfig().getVotePartyBungeeCommands()) runConsoleCommand(command);
+ for (String server : targets) sendVoteParty(server);
+ }
+ }
- if (!getConfig().getVotePartyBroadcast().isEmpty()) {
- broadcast(getConfig().getVotePartyBroadcast());
- }
+ private void persistRetainedVotePartyThreshold() {
+ try {
+ saveVotePartyStateDurably();
+ } catch (IOException failure) {
+ throw new IllegalStateException("Unable to retain the HTTP vote-party threshold", failure);
+ }
+ }
- for (String command : getConfig().getVotePartyBungeeCommands()) {
- runConsoleCommand(command);
- }
+ private Map onlineVotePartyTargets(Collection targets) {
+ Map online = new LinkedHashMap<>();
+ for (String server : targets) if (isSomeoneOnlineServerForVoteRouting(server))
+ online.putIfAbsent(server.toLowerCase(Locale.ROOT), server);
+ return online;
+ }
- if (getConfig().getVotePartySendToAllServers()) {
- for (String server : getAllAvailableServers()) {
- sendVoteParty(server);
- }
- } else {
- for (String server : getConfig().getVotePartyServersToSend()) {
- sendVoteParty(server);
- }
- }
+ private boolean canQueueVotePartyRewards(Map targets) {
+ for (String server : targets.keySet()) {
+ Collection pending = getVoteCachePendingVotePartyRewardIds(server);
+ if (pending != null && pending.size() >= MAX_PENDING_VOTE_PARTY_REWARDS) {
+ logSevere("HTTP vote-party reward backlog is full for " + targets.get(server)
+ + "; retaining the vote-party threshold");
+ return false;
}
- saveVoteCacheFile();
}
+ return true;
}
public abstract void debug(String str);
@@ -1154,6 +1564,16 @@ protected int[] getProjectedVotePartyState(int acceptedVotes) {
public abstract int getVoteCacheVotePartyIncreaseVotesRequired();
+ public abstract Collection getVoteCachePendingVotePartyServers();
+
+ public abstract Collection getVoteCachePendingVotePartyRewardIds(String server);
+
+ public abstract PendingVotePartyProxyEffects getVoteCachePendingVotePartyProxyEffects();
+
+ public abstract PendingVotePartyProxyEffects getVoteCacheQuarantinedVotePartyProxyEffects();
+
+ public abstract void saveVotePartyStateDurably() throws IOException;
+
public abstract boolean isPlayerOnline(String playerName);
/**
@@ -1248,6 +1668,7 @@ public void debug1(Throwable e) {
}
};
voteCacheHandler.load();
+ method = retainHttpForPendingDeliveries(method);
nonVotedPlayersCache = new NonVotedPlayersCache(getNonVotedCacheMySQLConfig(),
getConfig().getNonVotedCacheUseMySQL(), getConfig().getNonVotedCacheUseMainMySQL(),
@@ -1391,6 +1812,9 @@ public void sendMessage(String server, int delay, JsonEnvelope envelope) {
case SOCKETS:
sendSocketEnvelope(server, envelope);
break;
+ case HTTP:
+ sendGenericHttpEnvelope(server, envelope);
+ break;
default:
break;
}
@@ -1553,6 +1977,12 @@ public void onReceive(JsonEnvelope message) {
PRESENCE_MAINTENANCE_INTERVAL_SECONDS);
}
startControlServices();
+ // Open the listener last: backend callbacks can immediately reach routing,
+ // presence, vote-log, multi-proxy, and Control-adjacent runtime helpers.
+ if (method.equals(BungeeMethod.HTTP)) {
+ startHttpTransport();
+ }
+ scheduleVotePartyDeliveryRetry();
debug("VotingPluginProxy loaded, ONLINEMODE: " + getConfig().getOnlineMode());
}
@@ -1986,10 +2416,233 @@ public void triggerVote(String player, String service, boolean realVote, boolean
VoteTotalsSnapshot text, String uuid) {
vote(player, service, realVote, timeQueue, queueTime, text, uuid);
}
+
+ @Override
+ public void triggerVote(String player, String service, boolean realVote, boolean timeQueue, long queueTime,
+ VoteTotalsSnapshot text, String uuid, UUID voteId) {
+ receiveMultiProxyVote(player, service, realVote, timeQueue, queueTime, text, uuid, voteId);
+ }
+
+ @Override
+ public void triggerVote(String player, String service, boolean realVote, boolean timeQueue, long queueTime,
+ VoteTotalsSnapshot text, String uuid, UUID voteId, String origin) {
+ receiveMultiProxyVote(player, service, realVote, timeQueue, queueTime, text, uuid, voteId, origin);
+ }
+
+ @Override
+ public void onMultiProxyVoteAcknowledged(UUID voteId, String recipient) {
+ handleMultiProxyVoteAcknowledgement(voteId, recipient);
+ }
+
+ @Override
+ public void onMultiProxyVoteRetirementAcknowledged(UUID voteId, String recipient) {
+ handleMultiProxyVoteRetirementAcknowledgement(voteId, recipient);
+ }
+
+ @Override
+ public void onMultiProxyVoteRetirementRequested(UUID voteId, String origin) {
+ handleMultiProxyVoteRetirementRequest(voteId, origin);
+ }
};
multiProxyHandler.loadMultiProxySupport();
}
+ /** Receives and locally retries a forwarded vote under its wire-stable identity. */
+ protected synchronized void receiveMultiProxyVote(String player, String service, boolean realVote,
+ boolean timeQueue, long queueTime, VoteTotalsSnapshot totals, String uuid, UUID wireVoteId) {
+ receiveMultiProxyVote(player, service, realVote, timeQueue, queueTime, totals, uuid, wireVoteId, "");
+ }
+
+ /** Receives a reliable envelope and retains its sender only for a later ACK. */
+ protected synchronized void receiveMultiProxyVote(String player, String service, boolean realVote,
+ boolean timeQueue, long queueTime, VoteTotalsSnapshot totals, String uuid, UUID wireVoteId, String origin) {
+ UUID voteId = wireVoteId == null ? UUID.randomUUID() : wireVoteId;
+ if (completedMultiProxyVotes.containsKey(voteId)) {
+ acknowledgeCompletedMultiProxyVote(voteId, origin);
+ return;
+ }
+ if (getVoteCacheHandler().hasMultiProxyVoteCompletion(voteId)) {
+ rememberCompletedMultiProxyVote(voteId);
+ acknowledgeCompletedMultiProxyVote(voteId, origin);
+ return;
+ }
+ MultiProxyVoteRetry retry = multiProxyVoteRetries.get(voteId);
+ if (retry == null) {
+ if (multiProxyVoteRetries.size() >= MAX_MULTI_PROXY_VOTE_RETRIES) {
+ // Do not drop a sender delivery when the in-memory retry fence is full.
+ // The ordinary timed-vote cache is bounded by durable storage rather than
+ // this process heap and is loaded again after a restart. It also lets the
+ // normal queue processor drain the spill as retry slots become available.
+ if (retainForwardedVoteOverflow(player, service, realVote, totals, uuid, voteId, queueTime, origin)) {
+ scheduleTimeVoteRetry();
+ return;
+ }
+ logSevere("Unable to durably retain forwarded multi-proxy vote while the bounded retry queue is full");
+ return;
+ }
+ retry = new MultiProxyVoteRetry(player, service, realVote, timeQueue, queueTime, totals, uuid, voteId, origin);
+ multiProxyVoteRetries.put(voteId, retry);
+ }
+ if (!retry.scheduled) attemptMultiProxyVote(retry);
+ }
+
+ /** Durably spills an over-capacity forwarded vote into the normal replay queue. */
+ private boolean retainForwardedVoteOverflow(String player, String service, boolean realVote,
+ VoteTotalsSnapshot totals, String uuid, UUID voteId, long queueTime, String origin) {
+ if (voteId == null || player == null || service == null || uuid == null) return false;
+ for (VoteTimeQueue queued : getVoteCacheHandler().getTimeChangeQueue()) {
+ if (voteId.equals(queued.getVoteId())) return true;
+ }
+ long time = queueTime == 0L ? System.currentTimeMillis() : queueTime;
+ VoteTimeQueue queued = new VoteTimeQueue(voteId, player, service, time, false,
+ Collections.emptySet(), Collections.emptySet(), totals == null ? "" : totals.toString(), false, uuid);
+ queued.setRealVote(realVote);
+ queued.setMultiProxyOrigin(origin == null ? "" : origin);
+ return getVoteCacheHandler().addTimeVoteToCache(queued);
+ }
+
+ private static String[] decodeForwardedQueueTotals(String encoded) {
+ if (encoded == null || !encoded.startsWith(FORWARDED_QUEUE_TOTALS_PREFIX)) {
+ return new String[] { "true", encoded == null ? "" : encoded };
+ }
+ int flagIndex = FORWARDED_QUEUE_TOTALS_PREFIX.length();
+ int separator = flagIndex + 1;
+ if (encoded.length() <= separator || (encoded.charAt(flagIndex) != '0' && encoded.charAt(flagIndex) != '1')
+ || encoded.charAt(separator) != ':') {
+ return new String[] { "true", encoded };
+ }
+ char realVote = encoded.charAt(FORWARDED_QUEUE_TOTALS_PREFIX.length());
+ try {
+ String decoded = new String(Base64.getUrlDecoder().decode(encoded.substring(separator + 1)), StandardCharsets.UTF_8);
+ return new String[] { realVote == '1' ? "true" : "false", decoded };
+ } catch (IllegalArgumentException invalidEncoding) {
+ return new String[] { "true", encoded };
+ }
+ }
+
+ private synchronized void attemptMultiProxyVote(MultiProxyVoteRetry retry) {
+ attemptMultiProxyVote(retry, true);
+ }
+
+ private synchronized void attemptMultiProxyVote(MultiProxyVoteRetry retry, boolean allowSchedule) {
+ if (multiProxyVoteRetries.get(retry.voteId) != retry) return;
+ retry.scheduled = false;
+ if (!enabled) return;
+ if (retry.phase == MultiProxyVoteRetry.Phase.PERSIST_DEFERRED_RECEIPT) {
+ if (persistDeferredMultiProxyReceipt(retry)) {
+ multiProxyVoteRetries.remove(retry.voteId);
+ } else {
+ scheduleMultiProxyVoteRetry(retry, allowSchedule, "deferred-receipt persistence");
+ }
+ return;
+ }
+ if (retry.phase == MultiProxyVoteRetry.Phase.PERSIST_COMPLETION) {
+ if (getVoteCacheHandler().hasMultiProxyVoteCompletion(retry.voteId)
+ || getVoteCacheHandler().markMultiProxyVoteCompletedDurably(retry.voteId)) {
+ completeMultiProxyVote(retry);
+ } else {
+ scheduleMultiProxyVoteRetry(retry, allowSchedule, "completion-record persistence");
+ }
+ return;
+ }
+ try {
+ vote(retry.player, retry.service, retry.realVote, retry.timeQueue, retry.queueTime, retry.totals,
+ retry.uuid, retry.voteId);
+ if (retry.timeQueue && findUnprocessedQueuedVote(retry.voteId) != null) {
+ retry.phase = MultiProxyVoteRetry.Phase.PERSIST_DEFERRED_RECEIPT;
+ retry.attempts = 0;
+ if (persistDeferredMultiProxyReceipt(retry)) multiProxyVoteRetries.remove(retry.voteId);
+ else scheduleMultiProxyVoteRetry(retry, allowSchedule, "deferred-receipt persistence");
+ return;
+ }
+ retry.phase = MultiProxyVoteRetry.Phase.PERSIST_COMPLETION;
+ retry.attempts = 0;
+ if (getVoteCacheHandler().markMultiProxyVoteCompletedDurably(retry.voteId)) {
+ completeMultiProxyVote(retry);
+ } else {
+ scheduleMultiProxyVoteRetry(retry, allowSchedule, "completion-record persistence");
+ }
+ } catch (VoteRetryException retryable) {
+ scheduleMultiProxyVoteRetry(retry, allowSchedule, "durable-storage");
+ }
+ }
+
+ /** Binds a deferred receiver queue to its origin before dropping its live retry fence. */
+ private boolean persistDeferredMultiProxyReceipt(MultiProxyVoteRetry retry) {
+ VoteTimeQueue queued = findUnprocessedQueuedVote(retry.voteId);
+ if (queued == null) return false;
+ queued.setMultiProxyOrigin(retry.origin);
+ queued.setRealVote(retry.realVote);
+ queued.setDeliveryStateDirty(true);
+ return persistTimeVoteDelivery(queued);
+ }
+
+ private VoteTimeQueue findUnprocessedQueuedVote(UUID voteId) {
+ if (voteId == null) return null;
+ for (VoteTimeQueue queued : getVoteCacheHandler().getTimeChangeQueue()) {
+ if (voteId.equals(queued.getVoteId()) && !queued.isProcessed()) return queued;
+ }
+ return null;
+ }
+
+ private void completeMultiProxyVote(MultiProxyVoteRetry retry) {
+ multiProxyVoteRetries.remove(retry.voteId);
+ rememberCompletedMultiProxyVote(retry.voteId);
+ acknowledgeCompletedMultiProxyVote(retry.voteId, retry.origin);
+ }
+
+ private void acknowledgeCompletedMultiProxyVote(UUID voteId, String origin) {
+ if (origin == null || origin.isBlank() || multiProxyHandler == null) return;
+ multiProxyHandler.acknowledgeMultiProxyVote(voteId, origin);
+ }
+
+ private synchronized void handleMultiProxyVoteRetirementRequest(UUID voteId, String origin) {
+ if (voteId == null || origin == null || origin.isBlank() || multiProxyHandler == null) return;
+ for (VoteTimeQueue queued : getVoteCacheHandler().getTimeChangeQueue()) {
+ if (!voteId.equals(queued.getVoteId())) continue;
+ // A completion tombstone can be the only durable proof while deletion of
+ // the receiver's processed queue row is retrying. Retire that row first,
+ // otherwise deleting the tombstone would let the local queue replay it.
+ if (!queued.isProcessed() || !origin.equalsIgnoreCase(queued.getMultiProxyOrigin())
+ || !getVoteCacheHandler().removeTimeVote(queued)) return;
+ break;
+ }
+ if (!getVoteCacheHandler().removeMultiProxyVoteCompletion(voteId)) return;
+ removeCompletedMultiProxyVote(voteId);
+ multiProxyHandler.acknowledgeMultiProxyVoteRetirement(voteId, origin);
+ }
+
+ private void scheduleMultiProxyVoteRetry(MultiProxyVoteRetry retry, boolean allowSchedule, String reason) {
+ retry.attempts++;
+ ScheduledExecutorService scheduler = getScheduler();
+ if (!allowSchedule || retry.attempts >= MAX_MULTI_PROXY_VOTE_ATTEMPTS
+ || scheduler == null || scheduler.isShutdown()) {
+ if (!allowSchedule) return;
+ logSevere("Forwarded multi-proxy vote remains fenced after bounded " + reason + " retries for "
+ + MinecraftUsernameValidator.sanitizeForLog(retry.player));
+ return;
+ }
+ retry.scheduled = true;
+ try {
+ scheduler.schedule(retry, 5, TimeUnit.SECONDS);
+ } catch (RuntimeException schedulingFailure) {
+ retry.scheduled = false;
+ logSevere("Unable to schedule a forwarded multi-proxy vote retry; its live side-effect fence was retained");
+ }
+ }
+
+ private void rememberCompletedMultiProxyVote(UUID voteId) {
+ completedMultiProxyVotes.put(voteId, Boolean.TRUE);
+ while (completedMultiProxyVotes.size() > MAX_COMPLETED_MULTI_PROXY_VOTES) {
+ UUID oldest = completedMultiProxyVotes.keySet().iterator().next();
+ removeCompletedMultiProxyVote(oldest);
+ }
+ }
+
+ private void removeCompletedMultiProxyVote(UUID voteId) {
+ completedMultiProxyVotes.remove(voteId);
+ }
+
public abstract void log(String message);
/**
@@ -2378,9 +3031,26 @@ public void onDisable() {
/** Full runtime replacement waits for hosted workers; final proxy stop remains non-blocking. */
public void onDisable(boolean waitForHosted) {
+ invalidateDeferredHttpTransportReconciliation();
if (waitForHosted) {
prepareForRuntimeReplacement();
} else {
+ boolean liveRetriesSettled;
+ try {
+ liveRetriesSettled = settleMultiProxyAndLiveVotesForFinalShutdown();
+ } catch (RuntimeException failure) {
+ liveRetriesSettled = false;
+ }
+ if (!liveRetriesSettled) {
+ logSevere("Unable to durably quarantine live vote retries after bounded final-shutdown persistence attempts; operator reconciliation may be required after restart");
+ }
+ if (!quarantineInFlightVotePartyProxyCommandForReplacement()) {
+ awaitInFlightVotePartyProxyCommand();
+ if (!quarantineInFlightVotePartyProxyCommandForReplacement()) {
+ logSevere("Unable to durably quarantine an in-flight HTTP vote-party command after the bounded final-shutdown wait; operator reconciliation may be required after restart");
+ }
+ }
+ cancelPreparedHttpTransportChange();
controlServicesGeneration.incrementAndGet();
controlLifecycleExecutor.shutdownNow();
stopControlServices(false);
@@ -2388,8 +3058,130 @@ public void onDisable(boolean waitForHosted) {
completeRuntimeReplacementShutdown();
}
+ /** Settles consumed multi-proxy envelopes together with any live phase fences they own. */
+ protected synchronized boolean settleMultiProxyAndLiveVotesForFinalShutdown() {
+ for (int attempt = 0; attempt < FINAL_SHUTDOWN_PERSISTENCE_ATTEMPTS; attempt++) {
+ for (MultiProxyVoteRetry retry : new ArrayList<>(multiProxyVoteRetries.values())) {
+ retry.scheduled = false;
+ attemptMultiProxyVote(retry, false);
+ }
+ Set activeForwardedVotes = new LinkedHashSet<>();
+ for (UUID voteId : multiProxyVoteRetries.keySet()) {
+ if (liveVoteRetries.containsKey(voteId)) activeForwardedVotes.add(voteId);
+ }
+ boolean liveSettled = settleLiveVoteRetriesForFinalShutdown();
+ for (UUID voteId : activeForwardedVotes) {
+ if (!liveVoteRetries.containsKey(voteId)) {
+ MultiProxyVoteRetry retry = multiProxyVoteRetries.get(voteId);
+ if (retry != null) {
+ retry.phase = MultiProxyVoteRetry.Phase.PERSIST_COMPLETION;
+ retry.attempts = 0;
+ attemptMultiProxyVote(retry, false);
+ }
+ }
+ }
+ if (liveSettled && multiProxyVoteRetries.isEmpty()) return true;
+ }
+ // Platform disable callers always continue teardown. A receiver whose vote
+ // side effects completed but whose separate tombstone could not be written
+ // must therefore be represented by the ordinary durable queue, not retained
+ // only by this process's retry map.
+ if (liveVoteRetries.isEmpty() && quarantineCompletedMultiProxyRetriesForShutdown()) return true;
+ return liveVoteRetries.isEmpty() && multiProxyVoteRetries.isEmpty();
+ }
+
+ private boolean quarantineCompletedMultiProxyRetriesForShutdown() {
+ VoteCacheHandler cache = getVoteCacheHandler();
+ for (MultiProxyVoteRetry retry : new ArrayList<>(multiProxyVoteRetries.values())) {
+ if (retry.phase != MultiProxyVoteRetry.Phase.PERSIST_COMPLETION
+ || !retainCompletedMultiProxyRetry(cache, retry)) return false;
+ multiProxyVoteRetries.remove(retry.voteId);
+ }
+ return true;
+ }
+
+ /** Persists a completed receiver phase without acknowledging until its tombstone succeeds. */
+ private boolean retainCompletedMultiProxyRetry(VoteCacheHandler cache, MultiProxyVoteRetry retry) {
+ if (retry.voteId == null) return false;
+ for (VoteTimeQueue queued : cache.getTimeChangeQueue()) {
+ if (!retry.voteId.equals(queued.getVoteId())) continue;
+ // An unprocessed row owns deferred vote effects. Do not convert it into a
+ // completion fence: its origin is persisted by PERSIST_DEFERRED_RECEIPT.
+ if (!queued.isProcessed()) return false;
+ queued.setProcessed(true);
+ queued.setRealVote(retry.realVote);
+ queued.setMultiProxyOrigin(retry.origin);
+ queued.setMultiProxyCompletionPending(true);
+ queued.setDeliveryStateDirty(true);
+ return cache.updateTimeVote(queued);
+ }
+ VoteTimeQueue quarantine = new VoteTimeQueue(retry.voteId, retry.player, retry.service,
+ retry.queueTime == 0L ? System.currentTimeMillis() : retry.queueTime, false,
+ Collections.emptySet(), Collections.emptySet(), retry.totals == null ? "" : retry.totals.toString(), true,
+ retry.uuid);
+ quarantine.setRealVote(retry.realVote);
+ quarantine.setMultiProxyOrigin(retry.origin);
+ quarantine.setMultiProxyCompletionPending(true);
+ return cache.addTimeVoteToCache(quarantine);
+ }
+
+ /** Moves every in-memory live retry into the ordinary durable vote outbox before final teardown. */
+ protected synchronized boolean settleLiveVoteRetriesForFinalShutdown() {
+ if (liveVoteRetries.isEmpty()) return true;
+ VoteCacheHandler cache = getVoteCacheHandler();
+ boolean retained = true;
+ for (LiveVoteRetryState retry : liveVoteRetries.values()) {
+ if (retry.queuedVote != null) {
+ Queue queuedVotes = cache.getTimeChangeQueue();
+ if (queuedVotes == null || !queuedVotes.contains(retry.queuedVote)) {
+ retained &= cache.addTimeVoteToCache(retry.queuedVote);
+ }
+ }
+ Set onlineStates = Collections.newSetFromMap(new java.util.IdentityHashMap<>());
+ if (retry.rewardJournalOwner != null) onlineStates.add(retry.rewardJournalOwner);
+ if (retry.standaloneBroadcastState != null) onlineStates.add(retry.standaloneBroadcastState);
+ if (retry.pendingOnlineRewardState != null) onlineStates.add(retry.pendingOnlineRewardState);
+ for (OfflineBungeeVote state : onlineStates) {
+ retained &= cache.retainOnlineVoteForPersistenceRetry(state.getUuid(), state);
+ }
+ for (Map.Entry entry : retry.rewardStates.entrySet()) {
+ String server = entry.getKey();
+ for (String configured : getAllConfiguredServers()) {
+ if (configured.equalsIgnoreCase(server)) {
+ server = configured;
+ break;
+ }
+ }
+ retained &= cache.retainServerVoteForPersistenceRetry(server, entry.getValue());
+ }
+ }
+ for (int attempt = 0; attempt < FINAL_SHUTDOWN_PERSISTENCE_ATTEMPTS; attempt++) {
+ if (!retained || !cache.retryPendingVotePersistence()) continue;
+ for (Map.Entry entry : new ArrayList<>(liveVoteRetries.entrySet())) {
+ LiveVoteRetryState retry = entry.getValue();
+ QueuedVoteResult result = vote(retry.player, retry.service, retry.realVote, false, retry.time,
+ retry.totals, retry.uuid, retry.queuedVote, entry.getKey());
+ if (result == QueuedVoteResult.TERMINAL) liveVoteRetries.remove(entry.getKey());
+ }
+ if (liveVoteRetries.isEmpty()) return true;
+ }
+ return false;
+ }
+
/** Fail-closed gate that must complete before a replacement proxy runtime is created. */
- public void prepareForRuntimeReplacement() {
+ public synchronized void prepareForRuntimeReplacement() {
+ // Live retry markers fence side effects (including vote-party and totals) that
+ // have already run but whose durable outbox write has not. They are owned by
+ // this runtime only, so replacing it would let the scheduled listener retry
+ // start with an empty map and reapply those effects. Keep this runtime alive
+ // until the retry settles instead of dropping or replaying an uncertain vote.
+ if (!liveVoteRetries.isEmpty() || !multiProxyVoteRetries.isEmpty()) {
+ throw new IllegalStateException(
+ "Live vote retries or forwarded multi-proxy vote retries must settle before proxy runtime replacement");
+ }
+ if (!quarantineInFlightVotePartyProxyCommandForReplacement()) {
+ throw new IllegalStateException("In-flight vote-party command must be durably quarantined before proxy runtime replacement");
+ }
controlServicesGeneration.incrementAndGet();
synchronized (controlLifecycleLock) {
ControlConnector connector = controlConnector;
@@ -2399,10 +3191,12 @@ public void prepareForRuntimeReplacement() {
controlLifecycleExecutor.shutdown();
stopControlServicesLocked(true);
}
+ runtimeReplacementPrepared = true;
}
/** Best-effort remainder of runtime teardown after the Control overlap gate has succeeded. */
public void completeRuntimeReplacementShutdown() {
+ enabled = false;
cancelCommunicationTests("Proxy runtime stopped before the backend replied");
runCleanup("vote cache", () -> getVoteCacheHandler().saveVoteCache());
runCleanup("proxy MySQL messenger", () -> {
@@ -2418,6 +3212,7 @@ public void completeRuntimeReplacementShutdown() {
if (socketHandler != null) socketHandler.closeConnection();
});
runCleanup("socket clients", this::closeSocketClients);
+ runCleanup("HTTP transport", this::closeHttpTransport);
runCleanup("Redis subscriber", () -> {
if (redisHandler != null) redisHandler.close();
});
@@ -2436,7 +3231,6 @@ public void completeRuntimeReplacementShutdown() {
runCleanup("global data", () -> {
if (getGlobalDataHandler() != null) getGlobalDataHandler().shutdown();
});
- enabled = false;
}
private void runCleanup(String service, CleanupAction cleanup) {
@@ -2553,26 +3347,96 @@ private UUID parseUUIDFromString(String uuidAsString) {
}
public synchronized void processQueue() {
- while (getVoteCacheHandler().getTimeChangeQueue().size() > 0) {
- VoteTimeQueue vote = getVoteCacheHandler().getTimeChangeQueue().element();
- if (!vote.isProcessed()) {
- VoteTotalsSnapshot queuedTotals = vote.getTotals() == null || vote.getTotals().isEmpty() ? null
- : VoteTotalsSnapshot.parseStorage(vote.getTotals());
- QueuedVoteResult result = vote(vote.getName(), vote.getService(), true, false, vote.getTime(), queuedTotals,
- vote.getUuid(), vote);
- if (result == QueuedVoteResult.RETRY) {
+ java.util.Queue timeChangeQueue = getVoteCacheHandler().getTimeChangeQueue();
+ // Work from a bounded snapshot so an already-processed ACK outbox can remain
+ // durable without monopolizing the head of the rollover queue.
+ for (VoteTimeQueue vote : new ArrayList<>(timeChangeQueue)) {
+ if (!timeChangeQueue.contains(vote)) continue;
+ if (vote.isMultiProxyCompletionPending()) {
+ if (!getVoteCacheHandler().hasMultiProxyVoteCompletion(vote.getVoteId())
+ && !getVoteCacheHandler().markMultiProxyVoteCompletedDurably(vote.getVoteId())) {
scheduleTimeVoteRetry();
return;
}
- if (result == QueuedVoteResult.TERMINAL) {
- warn("Removing terminal rollover vote " + vote.getVoteId() + " for " + vote.getName() + "/"
- + ServiceSiteValidator.sanitizeForLog(vote.getService()));
- }
+ acknowledgeCompletedMultiProxyVote(vote.getVoteId(), vote.getMultiProxyOrigin());
+ if (!getVoteCacheHandler().removeTimeVote(vote)) {
+ scheduleTimeVoteRetry();
+ return;
+ }
+ continue;
+ }
+ if (!vote.isProcessed() && getVoteCacheHandler().hasTimeVoteCompletion(vote)) {
+ vote.setProcessed(true);
+ vote.setDeliveryStateDirty(true);
+ }
+ if (vote.isProcessed() && vote.hasPendingHttpBroadcastDeliveryIds()) {
+ // The reward/totals work is already complete. Only retry the durable
+ // standalone broadcasts; removing this row would lose their stable IDs.
+ retryPendingTimeBroadcasts();
+ if (vote.hasPendingHttpBroadcastDeliveryIds()) {
+ scheduleTimeVoteRetry();
+ continue;
+ }
+ }
+ if (vote.isProcessed() && vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) {
+ scheduleTimeVoteRetry();
+ return;
+ }
+ if (vote.isProcessed() && vote.isMultiProxyForwardingRequired()
+ && !vote.isMultiProxyForwardingHandled()) {
+ if (!retryDurableMultiProxyOutbox(vote)) {
+ scheduleTimeVoteRetry();
+ continue;
+ }
+ }
+ // A direct listener retry can still be queued after its ACK outbox completes.
+ // Keep that in-memory fence until the listener consumes it; queued-vote
+ // processing removes its own fence in vote() when no listener retry exists.
+ if (vote.isProcessed() && !vote.isMultiProxyForwardingRequired()) {
+ liveVoteRetries.remove(vote.getVoteId());
+ }
+ if (!vote.isProcessed()) {
+ String[] forwardedTotals = decodeForwardedQueueTotals(vote.getTotals());
+ boolean queuedRealVote = vote.isRealVote();
+ // Existing NUL-prefixed emergency rows predate the explicit realVote
+ // column. Decode them once for compatibility, while new rows preserve
+ // PostgreSQL-safe plain totals.
+ if (vote.getTotals() != null && vote.getTotals().startsWith(FORWARDED_QUEUE_TOTALS_PREFIX)) {
+ queuedRealVote = Boolean.parseBoolean(forwardedTotals[0]);
+ vote.setTotals(forwardedTotals[1]);
+ vote.setRealVote(queuedRealVote);
+ vote.setDeliveryStateDirty(true);
+ }
+ VoteTotalsSnapshot queuedTotals = forwardedTotals[1].isEmpty() ? null
+ : VoteTotalsSnapshot.parseStorage(forwardedTotals[1]);
+ QueuedVoteResult result = replayQueuedVote(vote, queuedTotals, queuedRealVote);
+ if (result == QueuedVoteResult.RETRY || result == QueuedVoteResult.RETRY_NONBLOCKING) {
+ scheduleTimeVoteRetry();
+ if (result == QueuedVoteResult.RETRY_NONBLOCKING) continue;
+ return;
+ }
+ if ((result == QueuedVoteResult.SUCCESS || result == QueuedVoteResult.TERMINAL)
+ && !vote.getMultiProxyOrigin().isBlank()) {
+ if (!getVoteCacheHandler().markMultiProxyVoteCompletedDurably(vote.getVoteId())) {
+ scheduleTimeVoteRetry();
+ return;
+ }
+ acknowledgeCompletedMultiProxyVote(vote.getVoteId(), vote.getMultiProxyOrigin());
+ }
+ if (result == QueuedVoteResult.TERMINAL) {
+ warn("Removing terminal rollover vote " + vote.getVoteId() + " for " + vote.getName() + "/"
+ + ServiceSiteValidator.sanitizeForLog(vote.getService()));
+ }
+ if (!getVoteCacheHandler().getTimeChangeQueue().contains(vote)) {
+ getVoteCacheHandler().clearTimeVoteCompletion(vote);
+ continue;
+ }
}
if (!getVoteCacheHandler().removeTimeVote(vote)) {
scheduleTimeVoteRetry();
return;
}
+ getVoteCacheHandler().clearTimeVoteCompletion(vote);
}
}
@@ -2604,10 +3468,14 @@ public void reloadFromControl() {
}
private void reloadRuntime(boolean restartControlServices) {
- method = BungeeMethod.getByName(getConfig().getBungeeMethod());
- if (getMethod() == null) {
- method = BungeeMethod.PLUGINMESSAGING;
- }
+ // A manual reload supersedes any nested deferred reload scheduled by an old
+ // runtime. The generation check prevents that task from rebuilding after
+ // shutdown or racing this explicit replacement.
+ invalidateDeferredHttpTransportReconciliation();
+ BungeeMethod configuredMethod = BungeeMethod.getByName(getConfig().getBungeeMethod());
+ if (configuredMethod == null) configuredMethod = BungeeMethod.PLUGINMESSAGING;
+ method = retainHttpForPendingDeliveries(configuredMethod);
+ scheduleDeferredHttpTransportReconciliation();
warnUnsupportedDedicatedVotingProxyMode();
if (!restartControlServices && method == BungeeMethod.SOCKETS) {
rebuildSocketClients();
@@ -2621,6 +3489,228 @@ private void reloadRuntime(boolean restartControlServices) {
}
}
+ private synchronized BungeeMethod retainHttpForPendingDeliveries(BungeeMethod configuredMethod) {
+ if (configuredMethod == BungeeMethod.HTTP && httpTransportServer != null
+ && !hasChangedLiveHttpConfiguration()) {
+ deferredHttpTransportReconciliation = false;
+ return configuredMethod;
+ }
+ HttpProxyTransportServer transport = httpTransportServer;
+ if (transport != null && httpTransportHasPendingDeliveries(transport)) {
+ persistLiveHttpListenerSettings();
+ deferredHttpTransportReconciliation = true;
+ logSevere("Retaining HTTP transport until durable deliveries are acknowledged");
+ return BungeeMethod.HTTP;
+ }
+ if (transport == null) {
+ try {
+ if (httpQueueHasPersistedDeliveries(
+ getDataFolderPlugin().toPath().resolve("http").resolve("outgoing-v1"))) {
+ retainedHttpStartupSettings = loadRetainedHttpListenerSettingsUnchecked();
+ deferredHttpTransportReconciliation = true;
+ logSevere("Retaining HTTP transport until persisted deliveries are acknowledged");
+ return BungeeMethod.HTTP;
+ }
+ } catch (IOException unreadableQueue) {
+ // An unreadable durable queue is not proof that it is empty. Reopen HTTP so
+ // its normal bounded loader can validate or recover the state.
+ retainedHttpStartupSettings = loadRetainedHttpListenerSettingsUnchecked();
+ deferredHttpTransportReconciliation = true;
+ logSevere("Retaining HTTP transport because its persisted delivery queue could not be inspected");
+ return BungeeMethod.HTTP;
+ }
+ }
+ if (hasPendingCachedHttpDeliveries()) {
+ if (transport != null) persistLiveHttpListenerSettings();
+ else retainedHttpStartupSettings = loadRetainedHttpListenerSettingsUnchecked();
+ deferredHttpTransportReconciliation = true;
+ logSevere("Retaining HTTP transport until cached deliveries are acknowledged");
+ return BungeeMethod.HTTP;
+ }
+ Collection servers = getVoteCachePendingVotePartyServers();
+ if (servers != null) {
+ for (String server : servers) {
+ Collection rewards = getVoteCachePendingVotePartyRewardIds(server);
+ if (rewards != null && !rewards.isEmpty()) {
+ if (transport != null) persistLiveHttpListenerSettings();
+ else retainedHttpStartupSettings = loadRetainedHttpListenerSettingsUnchecked();
+ deferredHttpTransportReconciliation = true;
+ logSevere("Retaining HTTP transport until pending vote-party rewards are acknowledged");
+ return BungeeMethod.HTTP;
+ }
+ }
+ }
+ deferredHttpTransportReconciliation = false;
+ retainedHttpStartupSettings = null;
+ if (configuredMethod != BungeeMethod.HTTP || hasChangedLiveHttpConfiguration()) {
+ clearRetainedHttpListenerSettings();
+ }
+ return configuredMethod;
+ }
+
+ private Path retainedHttpListenerSettingsPath() {
+ return getDataFolderPlugin().toPath().resolve("http").resolve("retained-listener-v1.bin");
+ }
+
+ private void persistLiveHttpListenerSettings() {
+ if (liveHttpHost == null || liveHttpPublicEndpoint == null || liveHttpPort <= 0) return;
+ try {
+ persistRetainedHttpListenerSettings(
+ new HttpListenerSettings(liveHttpHost, liveHttpPort, liveHttpPublicEndpoint));
+ } catch (IOException failure) {
+ throw new IllegalStateException("HTTP listener settings could not be retained for queued deliveries", failure);
+ }
+ }
+
+ private void persistRetainedHttpListenerSettings(HttpListenerSettings settings) throws IOException {
+ Path target = retainedHttpListenerSettingsPath();
+ Files.createDirectories(target.getParent());
+ ByteArrayOutputStream encoded = new ByteArrayOutputStream();
+ try (DataOutputStream output = new DataOutputStream(encoded)) {
+ output.writeUTF(HTTP_RETAINED_LISTENER_MAGIC);
+ output.writeUTF(settings.host);
+ output.writeInt(settings.port);
+ output.writeUTF(settings.publicEndpoint);
+ }
+ Path temporary = target.resolveSibling(target.getFileName() + "." + UUID.randomUUID() + ".tmp");
+ try {
+ Files.write(temporary, encoded.toByteArray(), StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
+ DurableFiles.publishStagedFile(temporary, target);
+ } finally {
+ Files.deleteIfExists(temporary);
+ }
+ }
+
+ private HttpListenerSettings loadRetainedHttpListenerSettingsUnchecked() {
+ try {
+ return loadRetainedHttpListenerSettings();
+ } catch (IOException failure) {
+ throw new IllegalStateException("Retained HTTP listener settings could not be read", failure);
+ }
+ }
+
+ private HttpListenerSettings loadRetainedHttpListenerSettings() throws IOException {
+ Path source = retainedHttpListenerSettingsPath();
+ if (!Files.isRegularFile(source)) return null;
+ try (DataInputStream input = new DataInputStream(Files.newInputStream(source))) {
+ if (!HTTP_RETAINED_LISTENER_MAGIC.equals(input.readUTF())) {
+ throw new IOException("Unsupported retained HTTP listener settings");
+ }
+ String host = input.readUTF();
+ int port = input.readInt();
+ String publicEndpoint = input.readUTF();
+ if (input.read() != -1 || host.isBlank() || port < 1 || port > 65535) {
+ throw new IOException("Invalid retained HTTP listener settings");
+ }
+ try {
+ validatedHttpEndpoint(publicEndpoint);
+ } catch (IllegalArgumentException invalid) {
+ throw new IOException("Invalid retained HTTP public endpoint", invalid);
+ }
+ return new HttpListenerSettings(host, port, publicEndpoint);
+ }
+ }
+
+ private void clearRetainedHttpListenerSettings() {
+ try {
+ DurableFiles.deleteIfExists(retainedHttpListenerSettingsPath());
+ } catch (IOException failure) {
+ logSevere("Unable to remove obsolete retained HTTP listener settings: " + failure.getMessage());
+ }
+ }
+
+ private boolean hasChangedLiveHttpConfiguration() {
+ return httpTransportServer != null && (!java.util.Objects.equals(liveHttpHost, getConfig().getHttpHost())
+ || liveHttpPort != getConfig().getHttpPort()
+ || !java.util.Objects.equals(liveHttpPublicEndpoint, getConfig().getHttpPublicEndpoint()));
+ }
+
+ /** Uses SimpleAPI #80 when deployed while remaining safe with an older published snapshot. */
+ protected boolean httpTransportHasPendingDeliveries(HttpProxyTransportServer transport) {
+ try {
+ return Boolean.TRUE.equals(transport.getClass().getMethod("hasPendingDeliveries").invoke(transport));
+ } catch (NoSuchMethodException unavailable) {
+ // Without an authoritative query, the live durable server cannot be proven empty.
+ return true;
+ } catch (ReflectiveOperationException | SecurityException failure) {
+ logSevere("Retaining HTTP transport because its live delivery queue could not be inspected");
+ return true;
+ }
+ }
+
+ /** Invokes the additive SimpleAPI disk probe, with an equivalent bounded compatibility fallback. */
+ protected boolean httpQueueHasPersistedDeliveries(java.nio.file.Path outgoingDirectory) throws IOException {
+ try {
+ Object result = HttpProxyTransportServer.class.getMethod("hasPersistedDeliveries", java.nio.file.Path.class)
+ .invoke(null, outgoingDirectory);
+ return Boolean.TRUE.equals(result);
+ } catch (NoSuchMethodException unavailable) {
+ return inspectPersistedHttpDeliveries(outgoingDirectory);
+ } catch (java.lang.reflect.InvocationTargetException failure) {
+ Throwable cause = failure.getCause();
+ if (cause instanceof IOException io) throw io;
+ throw new IOException("HTTP outgoing queue inspection failed", cause);
+ } catch (ReflectiveOperationException | SecurityException failure) {
+ throw new IOException("HTTP outgoing queue inspection failed", failure);
+ }
+ }
+
+ private boolean inspectPersistedHttpDeliveries(java.nio.file.Path outgoingDirectory) throws IOException {
+ java.nio.file.Path root = outgoingDirectory.toAbsolutePath().normalize();
+ if (!java.nio.file.Files.exists(root, java.nio.file.LinkOption.NOFOLLOW_LINKS)) return false;
+ if (java.nio.file.Files.isSymbolicLink(root)
+ || !java.nio.file.Files.isDirectory(root, java.nio.file.LinkOption.NOFOLLOW_LINKS))
+ throw new IOException("HTTP outgoing queue directory is invalid");
+ int backendCount = 0;
+ try (java.nio.file.DirectoryStream backends = java.nio.file.Files.newDirectoryStream(root)) {
+ for (java.nio.file.Path backend : backends) {
+ if (java.nio.file.Files.isSymbolicLink(backend)
+ || !java.nio.file.Files.isDirectory(backend, java.nio.file.LinkOption.NOFOLLOW_LINKS))
+ throw new IOException("HTTP outgoing queue contains an invalid entry");
+ if (++backendCount > 128) throw new IOException("HTTP outgoing queue exceeds its backend bound");
+ try (java.nio.file.DirectoryStream entries =
+ java.nio.file.Files.newDirectoryStream(backend)) {
+ if (entries.iterator().hasNext()) return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private boolean hasPendingCachedHttpDeliveries() {
+ VoteCacheHandler cache = getVoteCacheHandler();
+ if (cache == null) return false;
+ String[] cachedServers = cache.getCachedVotesServers();
+ if (cachedServers != null) {
+ for (String server : cachedServers) {
+ Collection votes = cache.getVotes(server);
+ if (hasPendingHttpDelivery(votes)) return true;
+ }
+ }
+ Collection onlinePlayers = cache.getOnlineVoteUUIDs();
+ if (onlinePlayers != null) {
+ for (String uuid : onlinePlayers) {
+ Collection votes = cache.getOnlineVotes(uuid);
+ if (hasPendingHttpDelivery(votes)) return true;
+ }
+ }
+ Collection timedVotes = cache.getTimeChangeQueue();
+ if (timedVotes != null) {
+ for (VoteTimeQueue vote : timedVotes) {
+ if (vote != null && vote.hasPendingHttpBroadcastDeliveryIds()) return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean hasPendingHttpDelivery(Collection votes) {
+ if (votes == null) return false;
+ for (OfflineBungeeVote vote : votes) {
+ if (vote != null && vote.hasPendingHttpDeliveryIds()) return true;
+ }
+ return false;
+ }
+
private synchronized void rebuildSocketClients() {
HashMap rebuilt = new HashMap<>();
try {
@@ -2653,6 +3743,384 @@ private synchronized boolean sendSocketEnvelope(String server, JsonEnvelope enve
}
}
+ protected synchronized boolean sendHttpEnvelope(String server, JsonEnvelope envelope) {
+ HttpProxyTransportServer transport = httpTransportServer;
+ return transport != null && transport.send(server, envelope);
+ }
+
+ /**
+ * Sends a vote envelope while recovering the stable ID exposed when durable
+ * publication is indeterminate. The first attempt has already published a
+ * quarantine file, so retrying the identical envelope with a new ID could
+ * deliver the vote twice.
+ */
+ protected boolean sendHttpEnvelopeWithRecovery(String server, JsonEnvelope envelope) {
+ return sendHttpEnvelopeWithRecovery(server, envelope, null);
+ }
+
+ /**
+ * Sends a non-reward HTTP message without allowing a transport handoff failure
+ * to retry the surrounding vote transaction. An ambiguous durable publication
+ * is recovered with its original ID; a definite rejection is reported and
+ * logged, rather than escaping through the void global-message adapter.
+ */
+ protected boolean sendGenericHttpEnvelope(String server, JsonEnvelope envelope) {
+ try {
+ boolean accepted = sendHttpEnvelopeWithRecovery(server, envelope);
+ if (!accepted) debug("HTTP transport rejected auxiliary delivery for " + server);
+ return accepted;
+ } catch (RuntimeException failure) {
+ debug("Unable to send HTTP auxiliary delivery: " + failure.getMessage());
+ return false;
+ }
+ }
+
+ protected boolean sendStableHttpEnvelope(String server, String deliveryId, JsonEnvelope envelope) {
+ try {
+ boolean accepted = sendHttpEnvelope(server, deliveryId, envelope);
+ if (!accepted) debug("HTTP transport rejected auxiliary delivery for " + server);
+ return accepted;
+ } catch (RuntimeException failure) {
+ debug("Unable to send HTTP auxiliary delivery: " + failure.getMessage());
+ return false;
+ }
+ }
+
+ protected boolean sendHttpEnvelopeWithRecovery(String server, JsonEnvelope envelope, OfflineBungeeVote cachedVote) {
+ String persistedId = cachedVote == null ? null : cachedVote.getHttpDeliveryId(server);
+ String stableId = persistedId != null ? persistedId
+ : stableCachedHttpDeliveryId("reward", server, envelope, cachedVote);
+ try {
+ boolean accepted = stableId == null ? sendHttpEnvelope(server, envelope)
+ : sendHttpEnvelope(server, stableId, envelope);
+ if (accepted && persistedId != null) {
+ cachedVote.setHttpDeliveryId(server, null);
+ cachedVote.setDeliveryStateDirty(true);
+ }
+ return accepted;
+ } catch (HttpProxyTransportServer.DeliveryRetryException failure) {
+ if (cachedVote != null) {
+ cachedVote.setHttpDeliveryId(server, failure.deliveryId());
+ cachedVote.setDeliveryStateDirty(true);
+ // Retry only after the caller durably records the recovered transport ID.
+ return false;
+ }
+ try {
+ boolean accepted = sendHttpEnvelope(server, failure.deliveryId(), envelope);
+ return accepted;
+ } catch (RuntimeException retryFailure) {
+ debug("Unable to recover HTTP vote delivery " + failure.deliveryId() + ": " + retryFailure.getMessage());
+ return false;
+ }
+ }
+ }
+
+ /** Derives the same transport identity after a crash before cache cleanup. */
+ private String stableCachedHttpDeliveryId(String purpose, String server, JsonEnvelope envelope,
+ OfflineBungeeVote cachedVote) {
+ if (cachedVote == null || server == null || envelope == null) return null;
+ String voteIdentity = cachedVote.getVoteId() == null ? stableCachedVoteRowIdentity(cachedVote)
+ : cachedVote.getVoteId().toString();
+ String key = "VotingPlugin:http-cache:v1\u0000" + purpose + "\u0000"
+ + server.toLowerCase(Locale.ROOT) + "\u0000" + envelope.getSubChannel() + "\u0000" + voteIdentity;
+ return UUID.nameUUIDFromBytes(key.getBytes(StandardCharsets.UTF_8)).toString();
+ }
+
+ /** Keeps a live vote's auxiliary broadcast idempotent across listener retries. */
+ private String stableLiveHttpBroadcastDeliveryId(UUID voteId, String server) {
+ String key = "VotingPlugin:http-live-broadcast:v1\u0000" + voteId + "\u0000"
+ + server.toLowerCase(Locale.ROOT);
+ return UUID.nameUUIDFromBytes(key.getBytes(StandardCharsets.UTF_8)).toString();
+ }
+
+ /**
+ * Distinguishes pre-vote-ID cache rows that can otherwise share every visible
+ * vote field. SQL primary keys and JSON entry keys are durable before this
+ * fallback is used; the tuple remains only for callers holding an unbound
+ * legacy object during a mixed-version transition.
+ */
+ private String stableCachedVoteRowIdentity(OfflineBungeeVote vote) {
+ if (vote.getServerVoteCacheRowId() > 0) return "server-sql:" + vote.getServerVoteCacheRowId();
+ if (vote.getOnlineVoteCacheRowId() > 0) return "online-sql:" + vote.getOnlineVoteCacheRowId();
+ if (vote.getServerVoteCacheJsonKey() != null) return "server-json:" + vote.getServerVoteCacheJsonKey();
+ if (vote.getOnlineVoteCacheJsonKey() != null) {
+ return "online-json:" + vote.getUuid() + ":" + vote.getOnlineVoteCacheJsonKey();
+ }
+ return "legacy:" + vote.getUuid() + "\u0000" + vote.getService() + "\u0000" + vote.getTime();
+ }
+
+ protected synchronized boolean sendHttpEnvelope(String server, String deliveryId, JsonEnvelope envelope) {
+ HttpProxyTransportServer transport = httpTransportServer;
+ return transport != null && transport.send(server, deliveryId, envelope);
+ }
+
+ private void startHttpTransport() {
+ try {
+ HttpListenerSettings startup = deferredHttpTransportReconciliation
+ ? retainedHttpStartupSettings : null;
+ if (startup == null) {
+ startup = new HttpListenerSettings(getConfig().getHttpHost(), getConfig().getHttpPort(),
+ getConfig().getHttpPublicEndpoint());
+ }
+ PreparedHttpTransport prepared = PREPARED_HTTP_TRANSPORTS.remove(httpTransportPreparationKey());
+ if (prepared != null && retainedHttpStartupSettings == null) {
+ if (!prepared.matches(getConfig())) {
+ prepared.close();
+ throw new IllegalStateException("Prepared HTTP transport does not match the installed configuration");
+ }
+ httpTransportServer = prepared.server;
+ httpEnrollmentAuthority = prepared.authority;
+ prepared.owner.set(this);
+ startup = new HttpListenerSettings(prepared.host, prepared.port, prepared.publicEndpoint);
+ } else {
+ if (prepared != null) prepared.close();
+ URI endpoint = validatedHttpEndpoint(startup.publicEndpoint);
+ File directory = new File(getDataFolderPlugin(), "http");
+ HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.toPath(), endpoint.getHost());
+ httpEnrollmentAuthority = new HttpEnrollmentAuthority(identity, directory.toPath());
+ httpTransportServer = new HttpProxyTransportServer(
+ new InetSocketAddress(startup.host, startup.port), identity,
+ httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"),
+ this::handleHttpTransportEnvelope, this::acknowledgeHttpDelivery);
+ httpTransportServer.start();
+ }
+ persistRetainedHttpListenerSettings(startup);
+ liveHttpHost = startup.host;
+ liveHttpPort = startup.port;
+ liveHttpPublicEndpoint = startup.publicEndpoint;
+ retainedHttpStartupSettings = null;
+ logInfo("HTTP transport listening securely on " + liveHttpHost + ":"
+ + httpTransportServer.port() + "; use /votingpluginproxy httpcode for each backend");
+ } catch (Exception failure) {
+ closeHttpTransport();
+ throw new IllegalStateException("HTTP transport could not start securely", failure);
+ }
+ }
+
+ /** Keeps the authenticated mTLS backend identity attached to security-sensitive proxy routing. */
+ protected void handleHttpTransportEnvelope(HttpProxyTransportServer.ReceivedEnvelope received) {
+ if (!isAuthenticatedHttpEnvelopeAllowed(received)) {
+ debug("Ignored HTTP envelope whose player-presence claim did not match its authenticated backend");
+ return;
+ }
+ GlobalMessageProxyHandler handler = globalMessageProxyHandler;
+ if (handler == null) throw new IllegalStateException("HTTP message router is not ready");
+ handler.onMessage(received.envelope());
+ }
+
+ private boolean isAuthenticatedHttpEnvelopeAllowed(HttpProxyTransportServer.ReceivedEnvelope received) {
+ if (received == null || received.envelope() == null || received.serverId() == null) return false;
+ String stampedServer = received.envelope().getFields().getOrDefault(VotingPluginWire.K_SERVER, "");
+ if (!received.serverId().equalsIgnoreCase(stampedServer)) return false;
+ if (!VotingPluginWire.SUB_LOGIN.equals(received.envelope().getSubChannel())) return true;
+ VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(received.envelope());
+ boolean modern = event.connectionId != null || event.backendIncarnationId != null
+ || event.backendStartedAt != 0L || event.presenceTimestamp != 0L;
+ if (!modern || isDedicatedVotingProxyEnabled()) return true;
+ // A player-facing proxy has a stronger authority than any backend: its live
+ // player connection supplies both the current route and (in online mode) UUID.
+ return isLegacyLoginDestinationAuthoritative(event.player, event.uuid, received.serverId());
+ }
+
+ private synchronized void closeHttpTransport() {
+ HttpProxyTransportServer transport = httpTransportServer;
+ httpTransportServer = null;
+ httpEnrollmentAuthority = null;
+ liveHttpHost = null;
+ liveHttpPort = 0;
+ liveHttpPublicEndpoint = null;
+ if (transport != null) transport.close();
+ }
+
+ /**
+ * Starts the candidate HTTP listener before Control publishes an HTTP method
+ * change. The replacement runtime adopts this listener, avoiding a bind/TLS/
+ * queue failure after the old runtime has already been torn down.
+ */
+ public synchronized void prepareHttpTransportChange(VotingPluginProxyConfig candidate) {
+ cancelPreparedHttpTransportChange();
+ PreparedHttpTransport prepared = createPreparedHttpTransport(candidate);
+ PREPARED_HTTP_TRANSPORTS.put(httpTransportPreparationKey(), prepared);
+ }
+
+ /** Returns whether the live listener already implements this HTTP snapshot. */
+ public synchronized boolean hasMatchingLiveHttpTransport(VotingPluginProxyConfig candidate) {
+ return candidate != null && httpTransportServer != null
+ && java.util.Objects.equals(liveHttpHost, candidate.getHttpHost())
+ && liveHttpPort == candidate.getHttpPort()
+ && java.util.Objects.equals(liveHttpPublicEndpoint, candidate.getHttpPublicEndpoint());
+ }
+
+ /** Returns whether a candidate would collide with the currently bound HTTP listener. */
+ public synchronized boolean hasLiveHttpBind(VotingPluginProxyConfig candidate) {
+ return candidate != null && httpTransportServer != null
+ && java.util.Objects.equals(liveHttpHost, candidate.getHttpHost())
+ && liveHttpPort == candidate.getHttpPort();
+ }
+
+ /** Cancels a candidate listener when configuration publication fails. */
+ public synchronized void cancelPreparedHttpTransportChange() {
+ PreparedHttpTransport prepared = PREPARED_HTTP_TRANSPORTS.remove(httpTransportPreparationKey());
+ if (prepared != null) prepared.close();
+ }
+
+ private PreparedHttpTransport createPreparedHttpTransport(VotingPluginProxyConfig candidate) {
+ HttpProxyTransportServer server = null;
+ try {
+ URI endpoint = validatedHttpEndpoint(candidate.getHttpPublicEndpoint());
+ File directory = new File(getDataFolderPlugin(), "http");
+ HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.toPath(), endpoint.getHost());
+ HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.toPath());
+ AtomicReference owner = new AtomicReference<>();
+ server = new HttpProxyTransportServer(
+ new InetSocketAddress(candidate.getHttpHost(), candidate.getHttpPort()), identity, authority,
+ directory.toPath().resolve("outgoing-v1"), received -> {
+ VotingPluginProxy active = owner.get();
+ if (active == null) throw new IllegalStateException("HTTP runtime replacement is not active");
+ active.handleHttpTransportEnvelope(received);
+ }, (backend, deliveryId) -> {
+ VotingPluginProxy active = owner.get();
+ if (active == null) throw new IOException("HTTP runtime replacement is not active");
+ active.acknowledgeHttpDelivery(backend, deliveryId);
+ });
+ server.start();
+ return new PreparedHttpTransport(server, authority, owner, candidate.getHttpHost(), candidate.getHttpPort(),
+ candidate.getHttpPublicEndpoint());
+ } catch (Exception failure) {
+ if (server != null) server.close();
+ throw new IllegalStateException("HTTP transport could not be prepared securely", failure);
+ }
+ }
+
+ private Path httpTransportPreparationKey() {
+ return getDataFolderPlugin().toPath().toAbsolutePath().normalize();
+ }
+
+ private URI validatedHttpEndpoint(String publicEndpoint) {
+ URI endpoint = URI.create(publicEndpoint);
+ if (!"https".equalsIgnoreCase(endpoint.getScheme()) || endpoint.getHost() == null
+ || endpoint.getPort() == 0 || endpoint.getPort() > 65535
+ || endpoint.getUserInfo() != null || endpoint.getQuery() != null || endpoint.getFragment() != null
+ || (endpoint.getPath() != null && !endpoint.getPath().isEmpty() && !"/".equals(endpoint.getPath()))) {
+ throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin");
+ }
+ return endpoint;
+ }
+
+ /**
+ * Receives every durable HTTP acknowledgement, including non-vote-party
+ * deliveries. The SimpleAPI queue removes the acknowledged entry immediately
+ * after this callback returns, so defer inspection to the proxy scheduler.
+ */
+ protected void acknowledgeHttpDelivery(String server, String deliveryId) throws IOException {
+ acknowledgeVotePartyDelivery(server, deliveryId);
+ scheduleDeferredHttpTransportReconciliation();
+ }
+
+ /** Rebuilds the proxy runtime only after the retained HTTP transport is proven empty. */
+ private void invalidateDeferredHttpTransportReconciliation() {
+ synchronized (this) {
+ httpTransportReconciliationGeneration++;
+ httpTransportReconciliationScheduled = false;
+ // Older work is fenced by its generation and must not suppress the
+ // replacement runtime's own single-flight probe.
+ httpTransportReconciliationRunning = false;
+ }
+ }
+
+ private void scheduleDeferredHttpTransportReconciliation() {
+ scheduleDeferredHttpTransportReconciliation(HTTP_TRANSPORT_RECONCILIATION_DELAY_MILLIS);
+ }
+
+ private void scheduleDeferredHttpTransportReconciliation(long delayMillis) {
+ scheduleDeferredHttpTransportReconciliation(delayMillis, -1L);
+ }
+
+ private void scheduleDeferredHttpTransportReconciliation(long delayMillis, long requiredGeneration) {
+ ScheduledExecutorService scheduler = getScheduler();
+ if (scheduler == null) return;
+ long generation;
+ synchronized (this) {
+ if (requiredGeneration >= 0L && requiredGeneration != httpTransportReconciliationGeneration) return;
+ if (!deferredHttpTransportReconciliation || method != BungeeMethod.HTTP
+ || httpTransportReconciliationScheduled || httpTransportReconciliationRunning) return;
+ httpTransportReconciliationScheduled = true;
+ generation = httpTransportReconciliationGeneration;
+ }
+ try {
+ scheduler.schedule(() -> reconcileDeferredHttpTransport(generation), delayMillis, TimeUnit.MILLISECONDS);
+ } catch (RuntimeException unavailable) {
+ synchronized (this) {
+ if (generation == httpTransportReconciliationGeneration)
+ httpTransportReconciliationScheduled = false;
+ }
+ debug("Unable to schedule deferred HTTP transport reconciliation: " + unavailable.getMessage());
+ }
+ }
+
+ private void reconcileDeferredHttpTransport(long generation) {
+ BungeeMethod configuredMethod;
+ boolean stillPending;
+ synchronized (this) {
+ if (generation != httpTransportReconciliationGeneration) return;
+ httpTransportReconciliationScheduled = false;
+ if (!enabled || !deferredHttpTransportReconciliation || method != BungeeMethod.HTTP) return;
+ configuredMethod = BungeeMethod.getByName(getConfig().getBungeeMethod());
+ if (configuredMethod == null) configuredMethod = BungeeMethod.PLUGINMESSAGING;
+ retainHttpForPendingDeliveries(configuredMethod);
+ stillPending = deferredHttpTransportReconciliation;
+ if (!stillPending) httpTransportReconciliationRunning = true;
+ }
+ if (stillPending) {
+ // The acknowledgement callback happens before queue removal. Re-arm the
+ // same single-flight probe so the final removal/cached-state clear is
+ // observed without relying on another inbound message.
+ scheduleDeferredHttpTransportReconciliation(HTTP_TRANSPORT_RECONCILIATION_POLL_MILLIS, generation);
+ return;
+ }
+ synchronized (this) {
+ if (generation != httpTransportReconciliationGeneration || !enabled) {
+ httpTransportReconciliationRunning = false;
+ return;
+ }
+ }
+ try {
+ // The concrete platform checks this generation again while holding its
+ // reload lock. That closes the gap between this scheduler callback and a
+ // manual platform reload/shutdown without introducing lock inversion.
+ reloadDeferredHttpTransportCore(generation);
+ } catch (RuntimeException reconciliationFailure) {
+ debug("Deferred HTTP transport reconciliation failed: " + reconciliationFailure.getMessage());
+ } finally {
+ boolean retry;
+ synchronized (this) {
+ retry = generation == httpTransportReconciliationGeneration && enabled
+ && deferredHttpTransportReconciliation && method == BungeeMethod.HTTP;
+ if (generation == httpTransportReconciliationGeneration)
+ httpTransportReconciliationRunning = false;
+ }
+ if (retry)
+ scheduleDeferredHttpTransportReconciliation(HTTP_TRANSPORT_RECONCILIATION_POLL_MILLIS, generation);
+ }
+ }
+
+ public String createHttpConnectionCode(String serverId) {
+ HttpEnrollmentAuthority authority = httpEnrollmentAuthority;
+ if (method != BungeeMethod.HTTP || authority == null) {
+ throw new IllegalStateException("The HTTP transport is not running");
+ }
+ String publicEndpoint = liveHttpPublicEndpoint != null
+ ? liveHttpPublicEndpoint : getConfig().getHttpPublicEndpoint();
+ return authority.createConnectionCode(serverId, URI.create(publicEndpoint), Duration.ofMinutes(15))
+ .encode();
+ }
+
+ public void revokeHttpBackend(String serverId) {
+ HttpEnrollmentAuthority authority = httpEnrollmentAuthority;
+ if (method != BungeeMethod.HTTP || authority == null) throw new IllegalStateException("The HTTP transport is not running");
+ authority.revoke(HttpTlsIdentity.canonicalServerId(serverId));
+ }
+
private synchronized void closeSocketClients() {
HashMap clients = clientHandles;
clientHandles = null;
@@ -2673,7 +4141,7 @@ static void stopSocketClients(Map clients) {
private void warnUnsupportedDedicatedVotingProxyMode() {
if (getConfig().getDedicatedVotingProxy() && (method == null || !method.supportsBackendPresence())) {
- logSevere("DedicatedVotingProxy requires MYSQL, REDIS, MQTT, or SOCKETS; PLUGINMESSAGING is disabled for "
+ logSevere("DedicatedVotingProxy requires MYSQL, REDIS, MQTT, SOCKETS, or HTTP; PLUGINMESSAGING is disabled for "
+ "dedicated-proxy routing. Falling back to normal proxy routing.");
}
}
@@ -2685,10 +4153,44 @@ private void warnUnsupportedDedicatedVotingProxyMode() {
public abstract void runConsoleCommand(String command);
+ /** Completion boundary used before durable HTTP vote-party command progress is advanced. */
+ protected CompletableFuture runVotePartyConsoleCommand(String command) {
+ runConsoleCommand(command);
+ return CompletableFuture.completedFuture(null);
+ }
+
+ /** Schedules the liveness fence for a platform command whose completion is uncertain. */
+ protected void scheduleVotePartyProxyCommandTimeout(Runnable timeout) {
+ CompletableFuture.delayedExecutor(60, TimeUnit.SECONDS).execute(timeout);
+ }
+
public abstract void saveVoteCacheFile();
public abstract void reloadCore(boolean mysql);
+ /** Platform implementations recheck this under their reload lock before replacing the runtime. */
+ protected void reloadDeferredHttpTransportCore(long generation) {
+ if (isDeferredHttpTransportGenerationCurrent(generation)) reloadCore(true);
+ }
+
+ protected synchronized boolean isDeferredHttpTransportGenerationCurrent(long generation) {
+ return enabled && generation == httpTransportReconciliationGeneration
+ && httpTransportReconciliationRunning && method == BungeeMethod.HTTP;
+ }
+
+ /** True when a changed configured transport must not retire this runtime's live HTTP queue yet. */
+ public synchronized boolean isRetainingHttpTransportForDeferredReconciliation() {
+ return deferredHttpTransportReconciliation && method == BungeeMethod.HTTP;
+ }
+
+ /** An active HTTP runtime changing method or endpoint needs a pre-teardown retention probe. */
+ public synchronized boolean requiresHttpRetentionCheckBeforeRuntimeReplacement() {
+ BungeeMethod configuredMethod = BungeeMethod.getByName(getConfig().getBungeeMethod());
+ if (configuredMethod == null) configuredMethod = BungeeMethod.PLUGINMESSAGING;
+ return method == BungeeMethod.HTTP
+ && (configuredMethod != BungeeMethod.HTTP || hasChangedLiveHttpConfiguration());
+ }
+
/** Strict Control reload path; failures propagate so the caller can restore its backup. */
public abstract void reloadControlConfiguration() throws Exception;
@@ -2856,9 +4358,303 @@ public void sendServerNameMessage() {
}
}
- public void sendVoteParty(String server) {
- if (isSomeoneOnlineServerForVoteRouting(server)) {
+ public synchronized void sendVoteParty(String server) {
+ if (!isSomeoneOnlineServerForVoteRouting(server)) return;
+ if (method != BungeeMethod.HTTP) {
globalMessageProxyHandler.sendMessage(server, 1, VotingPluginWire.votePartyBungee());
+ return;
+ }
+ Collection pending = getVoteCachePendingVotePartyRewardIds(server);
+ if (pending != null && pending.size() >= MAX_PENDING_VOTE_PARTY_REWARDS) {
+ logSevere("HTTP vote-party reward backlog is full for " + server);
+ return;
+ }
+ String deliveryId = UUID.randomUUID().toString();
+ // Persist intent before the bounded HTTP queue is attempted. A rejection or
+ // restart therefore leaves a retryable reward instead of silently losing it.
+ setVoteCachePendingVotePartyReward(server, deliveryId, true);
+ try {
+ saveVotePartyStateDurably();
+ } catch (IOException failure) {
+ setVoteCachePendingVotePartyReward(server, deliveryId, false);
+ throw new IllegalStateException("Unable to persist HTTP vote-party reward", failure);
+ }
+ retryPendingVotePartyRewards();
+ }
+
+ protected synchronized void retryPendingVotePartyRewards() {
+ if (!enabled || method != BungeeMethod.HTTP) return;
+ boolean retryRequired = false;
+ Collection servers = getVoteCachePendingVotePartyServers();
+ if (servers == null) return;
+ for (String server : new ArrayList<>(servers)) {
+ Collection pendingIds = getVoteCachePendingVotePartyRewardIds(server);
+ if (pendingIds == null || pendingIds.isEmpty()) continue;
+ String routingServer = resolveVotePartyRoutingServer(server);
+ for (String deliveryId : new ArrayList<>(pendingIds)) {
+ if (!isSomeoneOnlineServerForVoteRouting(routingServer)
+ || !sendHttpEnvelope(routingServer, deliveryId, VotingPluginWire.votePartyBungee())) {
+ retryRequired = true;
+ break;
+ }
+ retryRequired = true;
+ }
+ }
+ if (retryRequired) scheduleVotePartyDeliveryRetry();
+ }
+
+ protected synchronized boolean retryPendingVotePartyProxyEffects() {
+ if (!enabled) return true;
+ if (votePartyProxyCommandInFlight != 0L) return false;
+ if (votePartyProxyCommandCompletedUnpersisted
+ && !quarantineInFlightVotePartyProxyCommandForReplacement()) {
+ scheduleVotePartyDeliveryRetry();
+ return false;
+ }
+ PendingVotePartyProxyEffects pending;
+ try {
+ pending = getVoteCachePendingVotePartyProxyEffects();
+ } catch (RuntimeException invalid) {
+ logSevere("Pending HTTP vote-party proxy effects are invalid; retaining them without execution");
+ return false;
+ }
+ while (!pending.isEmpty()) {
+ PendingVotePartyProxyEffects remaining;
+ try {
+ if (!pending.broadcast().isEmpty()) {
+ broadcast(pending.broadcast());
+ remaining = new PendingVotePartyProxyEffects("", pending.commands());
+ } else {
+ CompletableFuture execution = runVotePartyConsoleCommand(pending.commands().get(0));
+ remaining = new PendingVotePartyProxyEffects("", pending.commands().subList(1, pending.commands().size()));
+ if (!execution.isDone()) {
+ long attempt = ++votePartyProxyCommandAttemptSequence;
+ if (attempt == 0L) attempt = ++votePartyProxyCommandAttemptSequence;
+ long commandAttempt = attempt;
+ votePartyProxyCommandInFlight = commandAttempt;
+ votePartyProxyCommandExecution = execution;
+ votePartyProxyCommandCompletedUnpersisted = false;
+ PendingVotePartyProxyEffects expected = pending;
+ PendingVotePartyProxyEffects completed = remaining;
+ execution.whenComplete((ignored, failure) ->
+ completeVotePartyProxyCommand(commandAttempt, expected, completed, failure));
+ long scheduledAttempt = commandAttempt;
+ try {
+ scheduleVotePartyProxyCommandTimeout(() ->
+ quarantineTimedOutVotePartyProxyCommand(scheduledAttempt, expected, completed, execution));
+ } catch (RuntimeException unavailable) {
+ logSevere("Unable to schedule the HTTP vote-party command liveness fence; the command remains pending");
+ }
+ return false;
+ }
+ execution.join();
+ }
+ } catch (RuntimeException failure) {
+ logSevere("A committed HTTP vote-party proxy effect failed and remains pending for retry");
+ scheduleVotePartyDeliveryRetry();
+ return false;
+ }
+
+ if (!persistVotePartyProxyEffectProgress(pending, remaining)) return false;
+ pending = remaining;
+ }
+ return true;
+ }
+
+ private void completeVotePartyProxyCommand(long attempt, PendingVotePartyProxyEffects expected,
+ PendingVotePartyProxyEffects remaining, Throwable failure) {
+ synchronized (this) {
+ if (votePartyProxyCommandInFlight != attempt) return;
+ votePartyProxyCommandInFlight = 0L;
+ votePartyProxyCommandExecution = null;
+ if (failure != null) {
+ logSevere("A committed HTTP vote-party proxy command failed and remains pending for retry");
+ if (enabled) scheduleVotePartyDeliveryRetry();
+ return;
+ }
+ PendingVotePartyProxyEffects current;
+ try {
+ current = getVoteCachePendingVotePartyProxyEffects();
+ } catch (RuntimeException invalid) {
+ logSevere("Pending HTTP vote-party proxy effects became invalid while a command was running");
+ return;
+ }
+ if (!current.equals(expected)) {
+ logSevere("Pending HTTP vote-party proxy effects changed while a command was running; progress was not advanced");
+ return;
+ }
+ if (!persistVotePartyProxyEffectProgress(expected, remaining)) {
+ votePartyProxyCommandCompletedUnpersisted = true;
+ if (!enabled) quarantineInFlightVotePartyProxyCommandForReplacement();
+ return;
+ }
+ votePartyProxyCommandCompletedUnpersisted = false;
+ if (enabled && retryPendingVotePartyProxyEffects()) {
+ if (method == BungeeMethod.HTTP) retryPendingVotePartyRewards();
+ if (votePartyVotes >= currentVotePartyVotesRequired) checkVoteParty();
+ }
+ }
+ }
+
+ private void quarantineTimedOutVotePartyProxyCommand(long attempt, PendingVotePartyProxyEffects expected,
+ PendingVotePartyProxyEffects remaining, CompletableFuture execution) {
+ synchronized (this) {
+ if (!enabled || execution.isDone() || votePartyProxyCommandInFlight != attempt) return;
+ PendingVotePartyProxyEffects current;
+ PendingVotePartyProxyEffects previousQuarantine;
+ try {
+ current = getVoteCachePendingVotePartyProxyEffects();
+ previousQuarantine = getVoteCacheQuarantinedVotePartyProxyEffects();
+ } catch (RuntimeException invalid) {
+ logSevere("Pending HTTP vote-party proxy effects became invalid while a command was running");
+ return;
+ }
+ if (!current.equals(expected)) {
+ logSevere("Pending HTTP vote-party proxy effects changed while a command was running; the attempt remains fenced");
+ return;
+ }
+ PendingVotePartyProxyEffects quarantine;
+ try {
+ java.util.List commands = new java.util.ArrayList<>(previousQuarantine.commands());
+ commands.add(expected.commands().get(0));
+ quarantine = new PendingVotePartyProxyEffects(previousQuarantine.broadcast(), commands);
+ } catch (RuntimeException full) {
+ logSevere("HTTP vote-party command quarantine is full; the uncertain command remains fenced");
+ return;
+ }
+ setVoteCacheQuarantinedVotePartyProxyEffects(quarantine);
+ setVoteCachePendingVotePartyProxyEffects(remaining);
+ try {
+ saveVotePartyStateDurably();
+ } catch (IOException | RuntimeException failure) {
+ setVoteCacheQuarantinedVotePartyProxyEffects(previousQuarantine);
+ setVoteCachePendingVotePartyProxyEffects(expected);
+ logSevere("Unable to durably quarantine an uncertain HTTP vote-party command; the attempt remains fenced");
+ return;
+ }
+ votePartyProxyCommandInFlight = 0L;
+ votePartyProxyCommandExecution = null;
+ logSevere("An HTTP vote-party proxy command did not complete within 60 seconds and was durably quarantined without retry");
+ if (retryPendingVotePartyProxyEffects()) {
+ if (method == BungeeMethod.HTTP) retryPendingVotePartyRewards();
+ if (votePartyVotes >= currentVotePartyVotesRequired) checkVoteParty();
+ }
+ }
+ }
+
+ protected synchronized boolean quarantineInFlightVotePartyProxyCommandForReplacement() {
+ if (votePartyProxyCommandInFlight == 0L && !votePartyProxyCommandCompletedUnpersisted) return true;
+ PendingVotePartyProxyEffects pending;
+ PendingVotePartyProxyEffects previousQuarantine;
+ try {
+ pending = getVoteCachePendingVotePartyProxyEffects();
+ previousQuarantine = getVoteCacheQuarantinedVotePartyProxyEffects();
+ if (pending.commands().isEmpty()) return false;
+ java.util.List commands = new java.util.ArrayList<>(previousQuarantine.commands());
+ commands.add(pending.commands().get(0));
+ PendingVotePartyProxyEffects quarantine = new PendingVotePartyProxyEffects(previousQuarantine.broadcast(), commands);
+ PendingVotePartyProxyEffects remaining = new PendingVotePartyProxyEffects("",
+ pending.commands().subList(1, pending.commands().size()));
+ setVoteCacheQuarantinedVotePartyProxyEffects(quarantine);
+ setVoteCachePendingVotePartyProxyEffects(remaining);
+ try {
+ saveVotePartyStateDurably();
+ } catch (IOException | RuntimeException failure) {
+ setVoteCacheQuarantinedVotePartyProxyEffects(previousQuarantine);
+ setVoteCachePendingVotePartyProxyEffects(pending);
+ return false;
+ }
+ votePartyProxyCommandInFlight = 0L;
+ votePartyProxyCommandExecution = null;
+ votePartyProxyCommandCompletedUnpersisted = false;
+ logSevere("An in-flight HTTP vote-party proxy command was durably quarantined for runtime replacement");
+ return true;
+ } catch (RuntimeException invalid) {
+ return false;
+ }
+ }
+
+ /** Gives an executing platform command a bounded chance to finish and persist progress before final teardown. */
+ protected void awaitInFlightVotePartyProxyCommand() {
+ CompletableFuture execution = votePartyProxyCommandExecution;
+ if (execution == null || execution.isDone()) return;
+ try {
+ execution.get(5, TimeUnit.SECONDS);
+ } catch (java.util.concurrent.ExecutionException ignored) {
+ // The completion callback retains failed commands for retry.
+ } catch (java.util.concurrent.TimeoutException timeout) {
+ logSevere("Timed out waiting for an in-flight HTTP vote-party command during final shutdown");
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ logSevere("Interrupted while waiting for an in-flight HTTP vote-party command during final shutdown");
+ }
+ }
+
+ private boolean persistVotePartyProxyEffectProgress(PendingVotePartyProxyEffects previous,
+ PendingVotePartyProxyEffects remaining) {
+ setVoteCachePendingVotePartyProxyEffects(remaining);
+ try {
+ saveVotePartyStateDurably();
+ return true;
+ } catch (IOException | RuntimeException failure) {
+ // Execution succeeded, but its progress was not durably confirmed. Restoring
+ // the marker gives at-least-once recovery rather than silently skipping it.
+ setVoteCachePendingVotePartyProxyEffects(previous);
+ logSevere("Unable to persist HTTP vote-party proxy-effect progress; the effect remains pending");
+ scheduleVotePartyDeliveryRetry();
+ return false;
+ }
+ }
+
+ private String resolveVotePartyRoutingServer(String canonicalServer) {
+ for (String configuredServer : getAllAvailableServers()) {
+ if (configuredServer.equalsIgnoreCase(canonicalServer)) return configuredServer;
+ }
+ return canonicalServer;
+ }
+
+ protected synchronized void acknowledgeVotePartyDelivery(String server, String deliveryId) throws IOException {
+ Collection pendingServers = getVoteCachePendingVotePartyServers();
+ if (pendingServers == null) return;
+ for (String pendingServer : new ArrayList<>(pendingServers)) {
+ if (!pendingServer.equalsIgnoreCase(server)) continue;
+ Collection pending = getVoteCachePendingVotePartyRewardIds(pendingServer);
+ if (pending == null || !pending.contains(deliveryId)) return;
+ setVoteCachePendingVotePartyReward(pendingServer, deliveryId, false);
+ try {
+ saveVotePartyStateDurably();
+ } catch (IOException | RuntimeException failure) {
+ setVoteCachePendingVotePartyReward(pendingServer, deliveryId, true);
+ if (failure instanceof IOException ioFailure) throw ioFailure;
+ throw (RuntimeException) failure;
+ }
+ return;
+ }
+ }
+
+ private void scheduleVotePartyDeliveryRetry() {
+ if (!enabled || votePartyDeliveryRetryScheduled || getScheduler() == null) return;
+ boolean pendingProxyEffects;
+ try {
+ pendingProxyEffects = !getVoteCachePendingVotePartyProxyEffects().isEmpty();
+ } catch (RuntimeException invalid) {
+ logSevere("Pending HTTP vote-party proxy effects are invalid; automatic execution is disabled");
+ return;
+ }
+ Collection pendingServers = method == BungeeMethod.HTTP ? getVoteCachePendingVotePartyServers() : null;
+ if (!pendingProxyEffects && (pendingServers == null || pendingServers.isEmpty())) return;
+ votePartyDeliveryRetryScheduled = true;
+ try {
+ getScheduler().schedule(() -> {
+ synchronized (VotingPluginProxy.this) { votePartyDeliveryRetryScheduled = false; }
+ if (retryPendingVotePartyProxyEffects()) {
+ if (method == BungeeMethod.HTTP) retryPendingVotePartyRewards();
+ if (votePartyVotes >= currentVotePartyVotesRequired) checkVoteParty();
+ }
+ }, 5, TimeUnit.SECONDS);
+ } catch (RuntimeException failure) {
+ votePartyDeliveryRetryScheduled = false;
+ debug("Unable to schedule HTTP vote-party reward retry: " + failure.getMessage());
}
}
@@ -2882,6 +4678,12 @@ public void setCurrentVotePartyVotes(int amount) {
public abstract void setVoteCacheVotePartyIncreaseVotesRequired(int votes);
+ public abstract void setVoteCachePendingVotePartyReward(String server, String deliveryId, boolean pending);
+
+ public abstract void setVoteCachePendingVotePartyProxyEffects(PendingVotePartyProxyEffects effects);
+
+ public abstract void setVoteCacheQuarantinedVotePartyProxyEffects(PendingVotePartyProxyEffects effects);
+
public void status() {
for (String s : getAllAvailableServers()) {
if (!isSomeoneOnlineServerForVoteRouting(s)) {
@@ -2987,15 +4789,21 @@ private static CommunicationTestResult failure(String server, BungeeMethod metho
private record PendingCommunicationTest(String server, BungeeMethod method, long startedAtNanos,
CompletableFuture result) { }
- private void sendVoteDelayRejected(String player, String uuid, String service, boolean playerOnline,
- String playerServer) {
+ private boolean sendVoteDelayRejected(UUID voteId, String player, String uuid, String service,
+ boolean playerOnline, String playerServer) {
if (!playerOnline || playerServer == null || !getAllAvailableServers().contains(playerServer)) {
debug("Not sending vote delay rejection for " + player + " because the player is offline");
- return;
+ return true;
}
- globalMessageProxyHandler.sendMessage(playerServer, 1,
- VotingPluginWire.voteDelayRejected(player, uuid, service, true));
+ JsonEnvelope envelope = VotingPluginWire.voteDelayRejected(player, uuid, service, true);
+ if (method == BungeeMethod.HTTP) {
+ String key = voteId + "\u0000vote-delay-rejected\u0000" + playerServer.toLowerCase(Locale.ROOT);
+ String deliveryId = UUID.nameUUIDFromBytes(key.getBytes(StandardCharsets.UTF_8)).toString();
+ return sendStableHttpEnvelope(playerServer, deliveryId, envelope);
+ }
+ globalMessageProxyHandler.sendMessage(playerServer, 1, envelope);
+ return true;
}
public String getWaitUntilDelaySiteFromService(String service) {
@@ -3123,16 +4931,42 @@ public boolean checkVoteDelay(String uuid, String player, String service, ArrayL
public synchronized void vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime,
VoteTotalsSnapshot text, String uuid) {
- vote(player, service, realVote, timeQueue, queueTime, text, uuid, null);
+ UUID voteId = UUID.randomUUID();
+ if (vote(player, service, realVote, timeQueue, queueTime, text, uuid, null, voteId) == QueuedVoteResult.RETRY) {
+ liveVoteRetries.remove(voteId);
+ throw new VoteRetryException();
+ }
+ }
+
+ public synchronized void vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime,
+ VoteTotalsSnapshot text, String uuid, UUID voteId) {
+ if (vote(player, service, realVote, timeQueue, queueTime, text, uuid, null, voteId) == QueuedVoteResult.RETRY) {
+ throw new VoteRetryException();
+ }
+ }
+
+ /** Releases retry-only state after the bounded event-listener retries are exhausted. */
+ public synchronized void abandonLiveVoteRetry(UUID voteId) {
+ if (voteId != null) liveVoteRetries.remove(voteId);
}
- private enum QueuedVoteResult {
- SUCCESS, RETRY, TERMINAL
+ enum QueuedVoteResult {
+ SUCCESS, RETRY, RETRY_NONBLOCKING, TERMINAL
+ }
+
+ protected QueuedVoteResult replayQueuedVote(VoteTimeQueue vote, VoteTotalsSnapshot totals, boolean realVote) {
+ return vote(vote.getName(), vote.getService(), realVote, false, vote.getTime(), totals, vote.getUuid(), vote);
}
private synchronized QueuedVoteResult vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime,
VoteTotalsSnapshot text, String uuid, VoteTimeQueue queuedVote) {
+ return vote(player, service, realVote, timeQueue, queueTime, text, uuid, queuedVote, null);
+ }
+
+ private synchronized QueuedVoteResult vote(String player, String service, boolean realVote, boolean timeQueue,
+ long queueTime, VoteTotalsSnapshot text, String uuid, VoteTimeQueue queuedVote, UUID requestedVoteId) {
try {
+ String requestPlayer = player;
if (!ServiceSiteValidator.isValid(service)) {
warn("Rejected vote with invalid service site '" + ServiceSiteValidator.sanitizeForLog(service) + "'");
return QueuedVoteResult.TERMINAL;
@@ -3143,11 +4977,28 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea
+ MinecraftUsernameValidator.sanitizeForLog(service) + "'");
return QueuedVoteResult.TERMINAL;
}
+ String requestIdentity = player.toLowerCase(Locale.ROOT) + "\u0000" + service.toLowerCase(Locale.ROOT);
+ // A platform listener can receive a vote in the small interval after the
+ // replacement gate succeeds and before the platform publishes its fresh
+ // runtime. Tell its bounded retry wrapper to retry against that fresh
+ // runtime rather than creating state that would be lost with this one.
+ if (runtimeReplacementPrepared) return QueuedVoteResult.RETRY;
UUID voteId = queuedVote == null ? null : queuedVote.getVoteId();
if (voteId == null) {
- voteId = UUID.randomUUID();
+ voteId = requestedVoteId == null
+ ? queuedVote == null ? UUID.randomUUID() : legacyTimedVoteId(queuedVote)
+ : requestedVoteId;
+ if (queuedVote != null && !getVoteCacheHandler().assignLegacyTimeVoteId(queuedVote, voteId)) {
+ warn("Unable to assign a stable ID to a legacy timed vote; retaining it for retry");
+ return QueuedVoteResult.RETRY;
+ }
}
+ LiveVoteRetryState retryState = liveVoteRetries.get(voteId);
+ if (retryState != null && !requestIdentity.equals(retryState.requestIdentity)) {
+ throw new IllegalArgumentException("Retry ID does not match the original vote");
+ }
+ boolean resumingAfterTotals = retryState != null;
// UUID resolution
if (!getConfig().getOnlineMode()) {
@@ -3215,24 +5066,29 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea
// Cache online state/server once (IMPORTANT for broadcast logic correctness)
final boolean playerOnline = isPlayerOnlineForVoteRouting(player);
final String playerServer = playerOnline ? getCurrentPlayerServerForVoteRouting(player) : null;
- long time = queueTime != 0 ? queueTime
- : LocalDateTime.now().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
+ long time = retryState != null ? retryState.time
+ : (queueTime != 0 ? queueTime
+ : LocalDateTime.now().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli());
Set broadcastTargets = queuedVote == null ? new LinkedHashSet<>()
: new LinkedHashSet<>(queuedVote.getBroadcastTargets());
Set broadcastForwardedServers = queuedVote == null ? new LinkedHashSet<>()
: new LinkedHashSet<>(queuedVote.getBroadcastForwardedServers());
+ if (retryState != null) {
+ broadcastForwardedServers.addAll(retryState.broadcastForwardedServers);
+ }
boolean proxyBroadcastHandled = queuedVote != null && queuedVote.isProxyBroadcastHandled();
boolean processesTotals = getConfig().getPrimaryServer() || !getConfig().getMultiProxySupport();
boolean managesTotals = processesTotals && getConfig().getBungeeManageTotals();
boolean canValidateStandaloneBroadcast = canForwardStandaloneBroadcast(managesTotals);
- ArrayList data = null;
+ ArrayList data = retryState == null ? null : retryState.totalsInput;
boolean queueForTimeChange = false;
// A completion callback can wipe totals and replay older queued votes. Run it
// before loading this vote's database snapshot so the calculations below use
// the post-rollover state.
- if (getConfig().getGlobalDataEnabled() && getGlobalDataHandler().isTimeChangedHappened()) {
+ if (!resumingAfterTotals && getConfig().getGlobalDataEnabled()
+ && getGlobalDataHandler().isTimeChangedHappened()) {
getGlobalDataHandler().checkForFinishedTimeChanges();
queueForTimeChange = timeQueue && getGlobalDataHandler().isTimeChangedHappened();
}
@@ -3240,7 +5096,7 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea
// Validate the vote before any immediate announcement. This keeps duplicate
// votes rejected by the delay check out of the GlobalData rollover queue and
// prevents announcing a vote that will not be processed.
- if (managesTotals) {
+ if (!resumingAfterTotals && managesTotals) {
if (getProxyMySQL() == null) {
logSevere("Mysql is not loaded correctly, stopping vote processing");
return QueuedVoteResult.RETRY;
@@ -3254,7 +5110,8 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea
data = getProxyMySQL().getExactQuery(new Column("uuid", new DataValueString(uuid)));
if (!checkVoteDelay(uuid, player, service, data, queuedVote == null)) {
log("Vote delay is not met for " + player + "/" + service + ", skipping vote");
- sendVoteDelayRejected(player, uuid, service, playerOnline, playerServer);
+ if (!sendVoteDelayRejected(voteId, player, uuid, service, playerOnline, playerServer)
+ && queuedVote != null) return QueuedVoteResult.RETRY;
return QueuedVoteResult.TERMINAL;
}
}
@@ -3271,6 +5128,7 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea
VoteTimeQueue delayedVote = new VoteTimeQueue(voteId, player, service, time,
proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers,
projectedTotals == null ? "" : projectedTotals.toString(), false, uuid);
+ delayedVote.setRealVote(realVote);
if (!getVoteCacheHandler().addTimeVoteToCache(delayedVote)) {
logSevere("Unable to persist queued rollover vote for " + player + "/" + service
+ "; skipping proxy broadcast");
@@ -3279,9 +5137,10 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea
if (proxyBroadcastHandled) {
for (String target : broadcastTargets) {
Set forwarded = sendProxyBroadcast(Collections.singleton(target), uuid, player,
- service, time, projectedTotals == null ? "" : projectedTotals.toString(), false);
- if (delayedVote.getBroadcastForwardedServers().addAll(forwarded)) {
- broadcastForwardedServers.addAll(forwarded);
+ service, time, projectedTotals == null ? "" : projectedTotals.toString(), false, delayedVote);
+ boolean newlyForwarded = delayedVote.getBroadcastForwardedServers().addAll(forwarded);
+ broadcastForwardedServers.addAll(forwarded);
+ if (newlyForwarded || delayedVote.isDeliveryStateDirty()) {
persistTimeVoteDelivery(delayedVote);
}
}
@@ -3291,10 +5150,32 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea
return QueuedVoteResult.SUCCESS;
}
- addVoteParty();
+ if (retryState == null) {
+ if (liveVoteRetries.size() >= MAX_LIVE_VOTE_RETRIES) return QueuedVoteResult.RETRY;
+ retryState = new LiveVoteRetryState();
+ retryState.requestIdentity = requestIdentity;
+ retryState.totalsInput = data;
+ retryState.player = requestPlayer;
+ retryState.service = service;
+ retryState.uuid = uuid;
+ retryState.time = time;
+ retryState.realVote = realVote;
+ liveVoteRetries.put(voteId, retryState);
+ }
+ if (queuedVote != null) {
+ retryState.queuedVote = queuedVote;
+ retryState.multiProxyForwardingHandled |= queuedVote.isMultiProxyForwardingHandled();
+ }
+ if (!retryState.votePartyApplied) {
+ // Fence the side effect before invoking it. If the call reports an
+ // indeterminate failure, a listener retry must not increment the party again.
+ retryState.votePartyApplied = true;
+ addVoteParty();
+ }
- // Totals processing (primary server OR no multiproxy)
- if (processesTotals) {
+ if (!retryState.totalsApplied) {
+ // Totals processing (primary server OR no multiproxy)
+ if (processesTotals) {
if (managesTotals) {
int allTimeTotal = getValue(data, "AllTimeTotal", 1);
int monthTotal = getValue(data, "MonthTotal", 1);
@@ -3340,10 +5221,20 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea
debug("Setting totals " + text.toString() + ", voteId=" + voteId + " for " + player + "/"
+ service);
+ retryState.totals = text;
+ retryState.totalsApplied = true;
getProxyMySQL().update(uuid, update);
} else {
text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0);
}
+ }
+ if (text == null) {
+ text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0);
+ }
+ retryState.totals = text;
+ retryState.totalsApplied = true;
+ } else {
+ text = retryState.totals;
}
if (text == null) {
text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0);
@@ -3352,7 +5243,14 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea
VoteLogStatus voteStatus = VoteLogStatus.IMMEDIATE;
boolean standaloneProxyBroadcast = canValidateStandaloneBroadcast && (proxyBroadcastHandled
|| proxyBroadcastDecider.usesImmediateForwarding(playerOnline));
+ if (getConfig().getSendVotesToAllServers() && retryState.rewardServers == null) {
+ retryState.rewardServers = new LinkedHashSet<>(getAllAvailableServers());
+ }
+ Set rewardServers = retryState.rewardServers == null
+ ? Collections.emptySet() : retryState.rewardServers;
Set proxyBroadcastTargets = Collections.emptySet();
+ OfflineBungeeVote standaloneBroadcastState = null;
+ boolean standaloneBroadcastStatePersisted = false;
if (standaloneProxyBroadcast) {
// A handled queued broadcast was necessarily sampled while the player was
// offline. Retry only targets that did not previously accept delivery.
@@ -3360,15 +5258,96 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea
: proxyBroadcastDecider.resolveTargets(false, null);
Set remainingTargets = new LinkedHashSet<>(proxyBroadcastTargets);
remainingTargets.removeAll(broadcastForwardedServers);
- broadcastForwardedServers.addAll(sendProxyBroadcast(remainingTargets, uuid, player, service, time,
- text == null ? "" : text.toString(), false));
+ standaloneBroadcastState = retryState.standaloneBroadcastState;
+ if (standaloneBroadcastState == null) {
+ standaloneBroadcastState = new OfflineBungeeVote(voteId, player, uuid, service, time, realVote,
+ text == null ? "" : text.toString(), false, true, proxyBroadcastTargets,
+ broadcastForwardedServers, !getConfig().getSendVotesToAllServers(), Collections.emptyMap(), queuedVote == null
+ ? Collections.emptyMap() : queuedVote.getHttpBroadcastDeliveryIds());
+ if (getConfig().getSendVotesToAllServers()) markRewardJournalTargets(standaloneBroadcastState, rewardServers);
+ retryState.standaloneBroadcastState = standaloneBroadcastState;
+ retryState.rewardJournalOwner = standaloneBroadcastState;
+ } else {
+ proxyBroadcastTargets = new LinkedHashSet<>(standaloneBroadcastState.getBroadcastTargets());
+ broadcastForwardedServers.addAll(standaloneBroadcastState.getBroadcastForwardedServers());
+ remainingTargets = new LinkedHashSet<>(proxyBroadcastTargets);
+ remainingTargets.removeAll(broadcastForwardedServers);
+ }
+ // The canonical broadcast journal must reach durable storage before the
+ // first target sees the message. Otherwise a proxy crash after one accepted
+ // send can permanently lose every remaining target.
+ standaloneBroadcastStatePersisted = persistAndSendStandaloneBroadcast(uuid,
+ standaloneBroadcastState, remainingTargets, broadcastForwardedServers);
+ retryState.broadcastForwardedServers.addAll(broadcastForwardedServers);
+ if (!standaloneBroadcastStatePersisted) {
+ logSevere("Unable to durably journal standalone broadcast for " + uuid);
+ return QueuedVoteResult.RETRY;
+ }
+ if (queuedVote != null) {
+ // Keep the source row's in-memory completion state aligned with the
+ // canonical broadcast journal. If persisting the final processed marker
+ // fails, processQueue can retry that marker without publishing an already
+ // accepted broadcast again.
+ queuedVote.getBroadcastForwardedServers().addAll(broadcastForwardedServers);
+ for (String forwardedServer : broadcastForwardedServers) {
+ queuedVote.setHttpBroadcastDeliveryId(forwardedServer, null);
+ }
+ }
}
// ===========================
// Send vote(s) to backend(s)
// ===========================
if (getConfig().getSendVotesToAllServers()) {
- for (String s : getAllAvailableServers()) {
+ OfflineBungeeVote rewardJournalOwner = retryState.rewardJournalOwner;
+ if (rewardJournalOwner == null) {
+ rewardJournalOwner = createCachedRewardVote(voteId, player, uuid, service, time, realVote,
+ text.toString(), false);
+ markRewardJournalTargets(rewardJournalOwner, rewardServers);
+ retryState.rewardJournalOwner = rewardJournalOwner;
+ if (!getVoteCacheHandler().addOnlineVoteDurably(uuid, rewardJournalOwner)) {
+ if (getVoteCacheHandler().retainOnlineVoteForPersistenceRetry(uuid, rewardJournalOwner)) {
+ scheduleCachedVoteDeliveryRetry();
+ }
+ return QueuedVoteResult.RETRY;
+ }
+ }
+ for (String server : rewardServers) {
+ if (retryState.deliveredRewardServers.contains(server)) continue;
+ OfflineBungeeVote rewardState = retryState.rewardStates.get(server.toLowerCase(Locale.ROOT));
+ if (rewardState == null) {
+ rewardState = createCachedRewardVote(voteId, player, uuid, service, time,
+ realVote, text.toString(), standaloneProxyBroadcast);
+ retryState.rewardStates.put(server.toLowerCase(Locale.ROOT), rewardState);
+ }
+ // Every target begins as pending so a crash before its send cannot lose
+ // the reward. Accepted deliveries are marked complete below.
+ rewardState.setRewardDelivered(false);
+ if (rewardState.getHttpDeliveryId(server) == null) {
+ rewardState.setHttpDeliveryId(server, rewardJournalDeliveryId(rewardJournalOwner, server));
+ }
+ if (!getVoteCacheHandler().addServerVoteDurably(server, rewardState)) {
+ if (getVoteCacheHandler().retainServerVoteForPersistenceRetry(server, rewardState)) {
+ scheduleCachedVoteDeliveryRetry();
+ }
+ logSevere("Unable to durably journal vote reward for " + server);
+ return QueuedVoteResult.RETRY;
+ }
+ }
+ if (!retryState.rewardJournalsDurable) {
+ // The source owner is removable only after every target-specific row is durable.
+ rewardJournalOwner.setRewardDelivered(true);
+ rewardJournalOwner.setDeliveryStateDirty(true);
+ if (!persistOnlineVoteDelivery(uuid, rewardJournalOwner)) return QueuedVoteResult.RETRY;
+ retryState.rewardJournalsDurable = true;
+ if (!rewardJournalOwner.isProxyBroadcastHandled() || rewardJournalOwner.isProxyBroadcastComplete()) {
+ if (!getVoteCacheHandler().tryRemoveOnlineVote(uuid, rewardJournalOwner)) {
+ scheduleCachedVoteDeliveryRetry();
+ }
+ }
+ }
+ for (String s : rewardServers) {
+ if (retryState.deliveredRewardServers.contains(s)) continue;
boolean forceCache = getConfig().getWaitForUserOnline()
&& (!playerOnline || playerServer == null || !playerServer.equalsIgnoreCase(s));
@@ -3379,12 +5358,6 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea
if ((!isSomeoneOnlineServerForVoteRouting(s) && method.requiresPlayerOnline()) || forceCache) {
voteStatus = VoteLogStatus.CACHED;
- boolean broadcastForwarded = standaloneProxyBroadcast
- && broadcastForwardedServers.containsAll(proxyBroadcastTargets);
- getVoteCacheHandler().addServerVote(s,
- new OfflineBungeeVote(voteId, player, uuid, service, time, realVote,
- text.toString(), broadcastForwarded, standaloneProxyBroadcast,
- proxyBroadcastTargets, broadcastForwardedServers, false));
debug("Caching vote for " + player + " on " + service + " for " + s);
} else {
boolean broadcastHere = !broadcastForwardedServers.contains(s);
@@ -3394,9 +5367,27 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea
broadcastHere = proxyBroadcastDecider.shouldBroadcast(s, targets);
}
- globalMessageProxyHandler.sendMessage(s, 2,
+ OfflineBungeeVote pendingVote = retryState.rewardStates.get(s.toLowerCase(Locale.ROOT));
+ boolean rewardAccepted = sendVoteEnvelopeAccepted(s, 2,
VotingPluginWire.vote(player, uuid, service, time, true, realVote, text.toString(),
- voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1));
+ voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1), pendingVote);
+ if (!rewardAccepted) {
+ pendingVote.setRewardDelivered(false);
+ pendingVote.setDeliveryStateDirty(true);
+ if (!persistServerVoteDelivery(s, pendingVote)) {
+ logSevere("Unable to persist the rejected vote delivery for " + s);
+ return QueuedVoteResult.RETRY;
+ }
+ voteStatus = VoteLogStatus.CACHED;
+ debug("Caching vote after the transport rejected delivery for " + s);
+ } else {
+ pendingVote.setRewardDelivered(true);
+ retryState.deliveredRewardServers.add(s);
+ pendingVote.setDeliveryStateDirty(true);
+ if (!persistServerVoteDelivery(s, pendingVote)) return QueuedVoteResult.RETRY;
+ getVoteCacheHandler().removeServerVotes(s,
+ new ArrayList<>(Collections.singletonList(pendingVote)));
+ }
}
}
} else {
@@ -3412,11 +5403,46 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea
broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets);
}
- globalMessageProxyHandler.sendMessage(server, 1,
- VotingPluginWire.voteOnline(player, uuid, service, time, true, realVote, text.toString(),
- voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1));
+ OfflineBungeeVote pendingVote = retryState.rewardStates.get(server.toLowerCase(Locale.ROOT));
+ if (pendingVote == null) {
+ pendingVote = standaloneProxyBroadcast
+ ? new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), false,
+ true, proxyBroadcastTargets, broadcastForwardedServers, false, Collections.emptyMap(),
+ standaloneBroadcastState.getHttpBroadcastDeliveryIds())
+ : createCachedRewardVote(voteId, player, uuid, service, time, realVote, text.toString(), false);
+ retryState.rewardStates.put(server.toLowerCase(Locale.ROOT), pendingVote);
+ }
+ boolean rewardAccepted = retryState.deliveredRewardServers.contains(server);
+ if (!rewardAccepted) {
+ rewardAccepted = sendVoteEnvelopeAccepted(server, 1,
+ VotingPluginWire.voteOnline(player, uuid, service, time, true, realVote, text.toString(),
+ voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1), pendingVote);
+ if (rewardAccepted) retryState.deliveredRewardServers.add(server);
+ }
+ if (!rewardAccepted) {
+ if (standaloneBroadcastState != null) {
+ standaloneBroadcastState.setRewardDelivered(false);
+ standaloneBroadcastState.setDeliveryStateDirty(true);
+ if (!persistOnlineVoteDelivery(uuid, standaloneBroadcastState)) {
+ return QueuedVoteResult.RETRY;
+ }
+ }
+ if (!getVoteCacheHandler().addOnlineVoteDurably(uuid, pendingVote)) {
+ if (getVoteCacheHandler().retainOnlineVoteForPersistenceRetry(uuid, pendingVote)) {
+ scheduleCachedVoteDeliveryRetry();
+ }
+ logSevere("Unable to durably cache the rejected online vote delivery for " + uuid);
+ return QueuedVoteResult.RETRY;
+ }
+ voteStatus = VoteLogStatus.CACHED;
+ standaloneBroadcastStatePersisted |= standaloneBroadcastState != null;
+ debug("Caching online vote after the transport rejected delivery for " + server);
+ } else if (standaloneBroadcastState != null
+ && standaloneBroadcastState.isProxyBroadcastComplete()) {
+ getVoteCacheHandler().removeOnlineVote(uuid, standaloneBroadcastState);
+ }
- if (canValidateStandaloneBroadcast && getConfig().getProxyBroadcastEnabled()
+ if (rewardAccepted && canValidateStandaloneBroadcast && getConfig().getProxyBroadcastEnabled()
&& !standaloneProxyBroadcast) {
Set targets = proxyBroadcastDecider.resolveTargets(true, playerServer);
@@ -3429,26 +5455,58 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea
if (getConfig().getBlockedServers().contains(targetServer)) {
continue;
}
+ if (broadcastForwardedServers.contains(targetServer)) {
+ continue;
+ }
- globalMessageProxyHandler.sendMessage(targetServer, bDelay,
- VotingPluginWire.voteBroadcast(uuid, player, service, time,
- text == null ? "" : text.toString(), true));
+ JsonEnvelope broadcast = VotingPluginWire.voteBroadcast(uuid, player, service, time,
+ text == null ? "" : text.toString(), true);
+ if (method == BungeeMethod.HTTP) {
+ String deliveryId = stableLiveHttpBroadcastDeliveryId(voteId, targetServer);
+ if (!sendStableHttpEnvelope(targetServer, deliveryId, broadcast)) {
+ retryState.broadcastForwardedServers.addAll(broadcastForwardedServers);
+ return queuedVote == null ? QueuedVoteResult.RETRY
+ : QueuedVoteResult.RETRY_NONBLOCKING;
+ }
+ broadcastForwardedServers.add(targetServer);
+ retryState.broadcastForwardedServers.add(targetServer);
+ } else {
+ globalMessageProxyHandler.sendMessage(targetServer, bDelay, broadcast);
+ }
bDelay++;
}
}
// multiproxy: envelope-only clear vote
- if (getConfig().getMultiProxySupport() && getConfig().getMultiProxyOneGlobalReward()) {
+ if (rewardAccepted && getConfig().getMultiProxySupport() && getConfig().getMultiProxyOneGlobalReward()) {
multiProxyHandler.sendClearVote(uuid, player);
}
} else {
voteStatus = VoteLogStatus.CACHED;
- boolean broadcastForwarded = standaloneProxyBroadcast
- && broadcastForwardedServers.containsAll(proxyBroadcastTargets);
- getVoteCacheHandler().addOnlineVote(uuid,
- new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(),
- broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets,
- broadcastForwardedServers, false));
+ if (standaloneBroadcastState != null) {
+ standaloneBroadcastState.setRewardDelivered(false);
+ standaloneBroadcastState.setDeliveryStateDirty(true);
+ if (!persistOnlineVoteDelivery(uuid, standaloneBroadcastState)) {
+ return QueuedVoteResult.RETRY;
+ }
+ }
+ OfflineBungeeVote cachedReward = standaloneProxyBroadcast
+ ? new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), false,
+ true, proxyBroadcastTargets, broadcastForwardedServers, false, Collections.emptyMap(),
+ standaloneBroadcastState.getHttpBroadcastDeliveryIds())
+ : createCachedRewardVote(voteId, player, uuid, service, time, realVote, text.toString(), false);
+ retryState.pendingOnlineRewardState = cachedReward;
+ boolean cachedDurably = getVoteCacheHandler().addOnlineVoteDurably(uuid, cachedReward);
+ if (!cachedDurably) {
+ if (getVoteCacheHandler().retainOnlineVoteForPersistenceRetry(uuid, cachedReward)) {
+ scheduleCachedVoteDeliveryRetry();
+ }
+ logSevere("Unable to durably cache online vote for " + uuid
+ + "; retaining it for persistence retry");
+ return QueuedVoteResult.RETRY;
+ }
+ retryState.pendingOnlineRewardState = null;
+ standaloneBroadcastStatePersisted |= standaloneBroadcastState != null;
debug("Caching online vote for " + player + " on " + service);
}
@@ -3460,6 +5518,12 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea
}
}
+ if (!persistUncachedStandaloneBroadcast(uuid, standaloneBroadcastState,
+ standaloneBroadcastStatePersisted)) {
+ logSevere("Unable to durably cache the pending standalone broadcast for " + uuid);
+ return QueuedVoteResult.RETRY;
+ }
+
// Vote logging
if (voteLogMysqlTable != null && getConfig().getVoteLoggingEnabled()) {
voteLogMysqlTable.logVote(voteId, voteStatus, service, uuid, player, time,
@@ -3469,11 +5533,17 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea
// ===========================
// Multiproxy forwarding
// ===========================
- if (getConfig().getMultiProxySupport() && getConfig().getPrimaryServer()) {
+ if (!retryState.multiProxyForwardingHandled && getConfig().getMultiProxySupport()
+ && getConfig().getPrimaryServer()
+ // An overflowed received envelope carries its sender in this field. It
+ // must be acknowledged after local completion, never forwarded again.
+ && (queuedVote == null || queuedVote.getMultiProxyOrigin().isBlank())) {
if (!getConfig().getMultiProxyOneGlobalReward()) {
debug("Sending global proxy vote envelope");
- multiProxyHandler.sendMultiProxyEnvelope(VotingPluginWire.vote(player, uuid, service, time, false,
- realVote, text == null ? "" : text.toString(), voteId, false, false, 1, 1));
+ if (!beginMultiProxyForwarding(retryState,
+ queuedVote == null ? retryState.queuedVote : queuedVote, player, uuid, service, time, realVote, text)) {
+ return QueuedVoteResult.RETRY;
+ }
} else {
// Only send to other proxies if the player DID NOT already receive reward on a
// backend
@@ -3486,28 +5556,528 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea
if (shouldSend) {
debug("Sending global proxy voteonline envelope");
- multiProxyHandler
- .sendMultiProxyEnvelope(VotingPluginWire.voteOnline(player, uuid, service, time, false,
- realVote, text == null ? "" : text.toString(), voteId, false, false, 1, 1));
+ if (!beginMultiProxyForwarding(retryState,
+ queuedVote == null ? retryState.queuedVote : queuedVote, player, uuid, service, time, realVote, text)) {
+ return QueuedVoteResult.RETRY;
+ }
} else {
debug("Not sending global proxy message for voteonline, player already got reward");
+ if (!markMultiProxyForwardingHandled(retryState, queuedVote)) return QueuedVoteResult.RETRY;
}
}
}
if (queuedVote != null) {
queuedVote.setProcessed(true);
- if (!getVoteCacheHandler().updateTimeVote(queuedVote)) {
- warn("Unable to persist completed rollover vote " + queuedVote.getVoteId()
- + "; attempting durable removal immediately");
+ if (!queuedVote.getMultiProxyOrigin().isBlank()) {
+ // Persist the receiver's completion phase with the processed fence. A
+ // failed tombstone write must resume completion, never skip to deletion.
+ queuedVote.setMultiProxyCompletionPending(true);
+ }
+ if (!persistTimeVoteDelivery(queuedVote)) {
+ if (!queuedVote.getMultiProxyOrigin().isBlank()) {
+ // The receiver may delete its queue row only after a durable completion
+ // fence exists. That tombstone also makes a sender retry idempotent.
+ if (getVoteCacheHandler().hasMultiProxyVoteCompletion(voteId)
+ || getVoteCacheHandler().markMultiProxyVoteCompletedDurably(voteId)) {
+ acknowledgeCompletedMultiProxyVote(voteId, queuedVote.getMultiProxyOrigin());
+ if (getVoteCacheHandler().removeTimeVote(queuedVote)) {
+ liveVoteRetries.remove(voteId);
+ return QueuedVoteResult.SUCCESS;
+ }
+ }
+ warn("Unable to persist completed forwarded vote " + queuedVote.getVoteId()
+ + "; retaining its receiver completion fence for retry");
+ return QueuedVoteResult.RETRY;
+ }
+ // Deleting the completed source row is itself a durable completion
+ // record. Prefer that fallback when updating the marker is unavailable.
+ if (getVoteCacheHandler().removeTimeVote(queuedVote)) {
+ liveVoteRetries.remove(voteId);
+ return QueuedVoteResult.SUCCESS;
+ }
+ if (getVoteCacheHandler().markTimeVoteCompletedDurably(queuedVote)) {
+ liveVoteRetries.remove(voteId);
+ return QueuedVoteResult.SUCCESS;
+ }
+ warn("Unable to persist or remove completed rollover vote " + queuedVote.getVoteId()
+ + "; retaining its live completion fence for retry");
+ return QueuedVoteResult.RETRY;
}
}
+ liveVoteRetries.remove(voteId);
return QueuedVoteResult.SUCCESS;
+ } catch (IllegalArgumentException e) {
+ throw e;
} catch (Exception e) {
e.printStackTrace();
return QueuedVoteResult.RETRY;
}
}
+ private UUID legacyTimedVoteId(VoteTimeQueue vote) {
+ return vote.legacyTimedVoteId();
+ }
+
+ /**
+ * Creates/persists the sender outbox before the first publish. Redis and socket
+ * publish APIs are fire-and-forget, so their return value is never a delivery
+ * acknowledgement. The record stays until every configured peer confirms its
+ * own durable completion by stable vote ID.
+ */
+ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTimeQueue queuedVote, String player,
+ String uuid, String service, long time, boolean realVote, VoteTotalsSnapshot totals) {
+ if (multiProxyHandler == null) return false;
+ // A corrupt or unpersistable peer-classification file cannot safely identify
+ // which targets are legacy. Stop before creating an ACK outbox that a legacy
+ // peer could never complete; the handler emits the operator recovery message.
+ if (multiProxyHandler.isMultiProxyVoteCapabilityRecoveryBlocked()) return false;
+ Set recipients = multiProxyHandler.getMultiProxyVoteRecipients();
+ Set renewingRecipients = multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal();
+ Set discoveringRecipients = multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery();
+ // Query discovery before advertising. A retry that reaches the durable
+ // deadline must classify the peer rather than sending a final handshake and
+ // immediately falling through to legacy; live discovery instead renews early
+ // with a bounded settling interval for asynchronous Redis replies.
+ if (!discoveringRecipients.isEmpty()) {
+ multiProxyHandler.renewMultiProxyVoteCapabilityDiscoveryIfDue();
+ } else if (!renewingRecipients.isEmpty()) {
+ multiProxyHandler.renewMultiProxyVoteCapabilityIfDue();
+ }
+ // Recording a first-observation deadline can itself fail and transition the
+ // handler into recovery-blocked state. Recheck before interpreting an empty
+ // discovery set as permission to use the lossy legacy route.
+ if (multiProxyHandler.isMultiProxyVoteCapabilityRecoveryBlocked()) return false;
+ Set configuredRecipients = multiProxyHandler.getConfiguredMultiProxyVoteRecipients();
+ // Keep custom MultiProxyHandler integrations that override only the
+ // established recipient method source-compatible while the base handler
+ // learns capabilities.
+ if (configuredRecipients.isEmpty() && !recipients.isEmpty()) configuredRecipients.addAll(recipients);
+ if (configuredRecipients.isEmpty()) return true;
+ if (!discoveringRecipients.isEmpty()) {
+ // Do not publish any part of this vote while a newly configured peer can
+ // still complete the capability handshake. Capturing only the known peers
+ // now would make the later legacy fallback hard to fence without a second,
+ // duplicate-prone outbox. The durable peer deadline bounds this retry even
+ // across a proxy restart.
+ if (admitMultiProxyDiscoveryOutbox(retryState, queuedVote, player, uuid, service, time, realVote, totals)
+ != null) scheduleTimeVoteRetry();
+ return false;
+ }
+ Set durableRecipients = new LinkedHashSet<>(recipients);
+ durableRecipients.addAll(renewingRecipients);
+ Set legacyRecipients = new LinkedHashSet<>(configuredRecipients);
+ legacyRecipients.removeAll(durableRecipients);
+ VoteTimeQueue outbox = queuedVote;
+ if (outbox != null && outbox.isMultiProxyCapabilityDiscoveryPending()
+ && !resolveMultiProxyDiscoveryOutbox(outbox)) return false;
+ if (durableRecipients.isEmpty()
+ && (outbox == null || !outbox.isMultiProxyForwardingRequired())) {
+ // Older peers do not understand acknowledgements. Preserve their historical
+ // fire-and-forget route instead of creating an outbox they can never ACK.
+ return multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.vote(player, uuid, service, time,
+ false, realVote, totals == null ? "" : totals.toString(), findLiveVoteId(retryState), false, false,
+ 1, 1), legacyRecipients);
+ }
+ if (outbox == null) {
+ outbox = new VoteTimeQueue(null, player, service, time, false, Collections.emptySet(),
+ Collections.emptySet(), totals == null ? "" : totals.toString(), true, uuid);
+ // The live retry key is the stable vote ID; copy it from the enclosing state
+ // by locating its identity rather than creating a new duplicate record.
+ outbox.setVoteId(findLiveVoteId(retryState));
+ outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), durableRecipients);
+ outbox.setMultiProxyLegacyPendingRecipients(legacyRecipients);
+ outbox.setRealVote(realVote);
+ outbox.setDeliveryStateDirty(true);
+ retryState.queuedVote = outbox;
+ if (outbox.getVoteId() == null || !getVoteCacheHandler().addTimeVoteToCache(outbox)) return false;
+ } else {
+ boolean alreadyQueued = false;
+ for (VoteTimeQueue candidate : getVoteCacheHandler().getTimeChangeQueue()) {
+ if (candidate != null && java.util.Objects.equals(outbox.getVoteId(), candidate.getVoteId())) {
+ alreadyQueued = true;
+ break;
+ }
+ }
+ if (!alreadyQueued && !getVoteCacheHandler().addTimeVoteToCache(outbox)) {
+ // A previous durable admission may have failed after the retry state kept
+ // this object in memory. Re-admit it before any publish;
+ // addTimeVoteToCache is idempotent for an already-durable queue entry.
+ return false;
+ }
+ }
+ if (outbox.isMultiProxyCapabilityDiscoveryPending()) {
+ // A crash can leave this row before classification. Resolve and persist the
+ // recipient split before publishing either route.
+ if (!resolveMultiProxyDiscoveryOutbox(outbox)) return false;
+ } else if (!outbox.isMultiProxyForwardingRequired()) {
+ outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), durableRecipients);
+ outbox.setMultiProxyLegacyPendingRecipients(legacyRecipients);
+ outbox.setRealVote(realVote);
+ outbox.setProcessed(true);
+ outbox.setDeliveryStateDirty(true);
+ if (!persistTimeVoteDelivery(outbox)) return false;
+ }
+ if (outbox.hasCompletedMultiProxyAcknowledgements()) {
+ return finishMultiProxyRetirement(retryState, outbox);
+ }
+ // A successful publish only means the transport accepted the invocation.
+ // Keep the durable row and wait for an acknowledgement before continuing.
+ if (!outbox.getMultiProxyLegacyPendingRecipients().isEmpty()) {
+ // Never retry this legacy copy as part of the ACK outbox: a legacy peer has
+ // no receiver dedupe/ACK contract, while capable peers stay fully durable.
+ // Clear the replayable set durably before publishing. If the transport
+ // rejects synchronously, restore it for retry; a process crash retains the
+ // historical legacy at-most-once behavior instead of duplicating rewards.
+ Set legacyPending = new LinkedHashSet<>(outbox.getMultiProxyLegacyPendingRecipients());
+ outbox.setMultiProxyLegacyPendingRecipients(Collections.emptySet());
+ outbox.setDeliveryStateDirty(true);
+ if (!persistTimeVoteDelivery(outbox)) return false;
+ if (!multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.vote(player, uuid, service, time,
+ false, realVote, totals == null ? "" : totals.toString(), outbox.getVoteId(), false, false, 1, 1),
+ legacyPending)) {
+ outbox.setMultiProxyLegacyPendingRecipients(legacyPending);
+ outbox.setDeliveryStateDirty(true);
+ persistTimeVoteDelivery(outbox);
+ return false;
+ }
+ }
+ if (outbox.getMultiProxyRecipients().isEmpty()) {
+ // Discovery resolved every peer to the legacy route. There can be no ACK
+ // retirement phase, so retire the durable intent after the one-way copy was
+ // durably marked accepted.
+ boolean handled = markMultiProxyForwardingHandled(retryState, outbox);
+ if (handled) scheduleTimeVoteRetry();
+ return handled;
+ }
+ if (!sendDurableMultiProxyOutbox(outbox)) return false;
+ scheduleTimeVoteRetry();
+ return false;
+ }
+
+ /**
+ * Persists a forwarding intent before the first discovery-induced retry. The
+ * local vote effects have already completed at this point; without this row a
+ * shutdown between the retry result and the next listener attempt loses the
+ * only record that the vote still needs forwarding.
+ */
+ private VoteTimeQueue admitMultiProxyDiscoveryOutbox(LiveVoteRetryState retryState, VoteTimeQueue queuedVote,
+ String player, String uuid, String service, long time, boolean realVote, VoteTotalsSnapshot totals) {
+ VoteTimeQueue outbox = queuedVote;
+ if (outbox == null) {
+ outbox = new VoteTimeQueue(null, player, service, time, false, Collections.emptySet(),
+ Collections.emptySet(), totals == null ? "" : totals.toString(), true, uuid);
+ outbox.setVoteId(findLiveVoteId(retryState));
+ if (outbox.getVoteId() == null) return null;
+ retryState.queuedVote = outbox;
+ }
+ outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), Collections.emptySet());
+ outbox.setMultiProxyCapabilityDiscoveryPending(true);
+ outbox.setMultiProxyLegacyPendingRecipients(Collections.emptySet());
+ outbox.setRealVote(realVote);
+ outbox.setProcessed(true);
+ outbox.setDeliveryStateDirty(true);
+ boolean alreadyQueued = false;
+ for (VoteTimeQueue candidate : getVoteCacheHandler().getTimeChangeQueue()) {
+ if (candidate != null && java.util.Objects.equals(outbox.getVoteId(), candidate.getVoteId())) {
+ alreadyQueued = true;
+ break;
+ }
+ }
+ if (!alreadyQueued) return getVoteCacheHandler().addTimeVoteToCache(outbox) ? outbox : null;
+ return persistTimeVoteDelivery(outbox) ? outbox : null;
+ }
+
+ /** Resolves a restored discovery-pending outbox and persists its recipient split before sending. */
+ private boolean resolveMultiProxyDiscoveryOutbox(VoteTimeQueue outbox) {
+ if (multiProxyHandler == null || multiProxyHandler.isMultiProxyVoteCapabilityRecoveryBlocked()) return false;
+ Set recipients = multiProxyHandler.getMultiProxyVoteRecipients();
+ Set renewingRecipients = multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal();
+ Set discoveringRecipients = multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery();
+ if (!discoveringRecipients.isEmpty()) {
+ multiProxyHandler.renewMultiProxyVoteCapabilityDiscoveryIfDue();
+ } else if (!renewingRecipients.isEmpty()) {
+ multiProxyHandler.renewMultiProxyVoteCapabilityIfDue();
+ }
+ if (multiProxyHandler.isMultiProxyVoteCapabilityRecoveryBlocked() || !discoveringRecipients.isEmpty()) return false;
+ Set configuredRecipients = multiProxyHandler.getConfiguredMultiProxyVoteRecipients();
+ if (configuredRecipients.isEmpty() && !recipients.isEmpty()) configuredRecipients.addAll(recipients);
+ Set durableRecipients = new LinkedHashSet<>(recipients);
+ durableRecipients.addAll(renewingRecipients);
+ Set legacyRecipients = new LinkedHashSet<>(configuredRecipients);
+ legacyRecipients.removeAll(durableRecipients);
+ outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), durableRecipients);
+ outbox.setMultiProxyLegacyPendingRecipients(legacyRecipients);
+ outbox.setMultiProxyCapabilityDiscoveryPending(false);
+ outbox.setDeliveryStateDirty(true);
+ return persistTimeVoteDelivery(outbox);
+ }
+
+ private UUID findLiveVoteId(LiveVoteRetryState retryState) {
+ for (Map.Entry entry : liveVoteRetries.entrySet()) {
+ if (entry.getValue() == retryState) return entry.getKey();
+ }
+ return null;
+ }
+
+ /** Re-sends an unacknowledged outbox using the exact persisted stable ID. */
+ private boolean retryDurableMultiProxyOutbox(VoteTimeQueue outbox) {
+ if (outbox.isMultiProxyCapabilityDiscoveryPending() && !resolveMultiProxyDiscoveryOutbox(outbox)) return false;
+ if (outbox.hasCompletedMultiProxyAcknowledgements()) {
+ return finishMultiProxyRetirement(null, outbox);
+ }
+ if (!outbox.getMultiProxyLegacyPendingRecipients().isEmpty()) {
+ Set legacyPending = new LinkedHashSet<>(outbox.getMultiProxyLegacyPendingRecipients());
+ outbox.setMultiProxyLegacyPendingRecipients(Collections.emptySet());
+ outbox.setDeliveryStateDirty(true);
+ if (!persistTimeVoteDelivery(outbox)) return false;
+ if (!multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.vote(outbox.getName(),
+ outbox.getUuid(), outbox.getService(), outbox.getTime(), false, outbox.isRealVote(),
+ outbox.getTotals(), outbox.getVoteId(), false, false, 1, 1), legacyPending)) {
+ outbox.setMultiProxyLegacyPendingRecipients(legacyPending);
+ outbox.setDeliveryStateDirty(true);
+ persistTimeVoteDelivery(outbox);
+ return false;
+ }
+ }
+ if (outbox.getMultiProxyRecipients().isEmpty()) {
+ boolean handled = markMultiProxyForwardingHandled(null, outbox);
+ if (handled) scheduleTimeVoteRetry();
+ return handled;
+ }
+ sendDurableMultiProxyOutbox(outbox);
+ return false;
+ }
+
+ private boolean sendDurableMultiProxyOutbox(VoteTimeQueue outbox) {
+ if (multiProxyHandler == null || outbox.getVoteId() == null || outbox.getMultiProxyOrigin().isBlank()) return false;
+ Set pending = new LinkedHashSet<>(outbox.getMultiProxyRecipients());
+ pending.removeAll(outbox.getMultiProxyAcknowledgedServers());
+ if (pending.isEmpty()) return true;
+ // A durable outbox recipient is known to have required acknowledgements when
+ // the row was created. It must therefore have a *current* capability lease
+ // before a retry publishes to it. In particular, a proxy restart/reload clears
+ // in-memory capability knowledge while the outbox survives; treating that
+ // recipient as legacy would let a rolled-back peer execute every retry without
+ // ever acknowledging it.
+ Set leasedRecipients = new LinkedHashSet<>();
+ for (String recipient : multiProxyHandler.getMultiProxyVoteRecipients()) {
+ if (recipient != null) leasedRecipients.add(recipient.toLowerCase(Locale.ROOT));
+ }
+ Set awaitingRenewal = new LinkedHashSet<>();
+ for (String recipient : pending) {
+ if (recipient != null && !leasedRecipients.contains(recipient.toLowerCase(Locale.ROOT))) {
+ awaitingRenewal.add(recipient.toLowerCase(Locale.ROOT));
+ }
+ }
+ if (!awaitingRenewal.isEmpty()) {
+ // Queue retries may be the only activity after an ACK was lost. Renew the
+ // handshake here so a recovered peer can become eligible without waiting
+ // for an unrelated new vote, while the handler bounds advertisements.
+ multiProxyHandler.renewMultiProxyVoteCapabilityIfDue();
+ }
+ pending.removeIf(recipient -> recipient != null
+ && awaitingRenewal.contains(recipient.toLowerCase(Locale.ROOT)));
+ if (pending.isEmpty()) {
+ // This method is also reached directly from the listener path, where no
+ // enclosing queue loop schedules the next lease-renewal attempt. Keep the
+ // durable row live until the peer answers the bounded handshake.
+ scheduleTimeVoteRetry();
+ return false;
+ }
+ return multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVote(outbox.getName(),
+ outbox.getUuid(), outbox.getService(), outbox.getTime(), false, outbox.isRealVote(), outbox.getTotals(),
+ outbox.getVoteId(), false, false, 1, 1, outbox.getMultiProxyOrigin()),
+ pending);
+ }
+
+ /** Handles an ACK only after confirming it belongs to a configured recipient. */
+ private synchronized void handleMultiProxyVoteAcknowledgement(UUID voteId, String recipient) {
+ if (voteId == null || recipient == null || recipient.isBlank()) return;
+ LiveVoteRetryState state = liveVoteRetries.get(voteId);
+ VoteTimeQueue outbox = state == null ? null : state.queuedVote;
+ if (outbox == null) {
+ for (VoteTimeQueue candidate : getVoteCacheHandler().getTimeChangeQueue()) {
+ if (voteId.equals(candidate.getVoteId()) && candidate.isMultiProxyForwardingRequired()) {
+ outbox = candidate;
+ break;
+ }
+ }
+ }
+ if (outbox == null || !outbox.acknowledgeMultiProxyRecipient(recipient)) return;
+ outbox.setDeliveryStateDirty(true);
+ if (!persistTimeVoteDelivery(outbox)) {
+ scheduleTimeVoteRetry();
+ return;
+ }
+ scheduleTimeVoteRetry();
+ }
+
+ private synchronized void handleMultiProxyVoteRetirementAcknowledgement(UUID voteId, String recipient) {
+ if (voteId == null || recipient == null || recipient.isBlank()) return;
+ LiveVoteRetryState state = liveVoteRetries.get(voteId);
+ VoteTimeQueue outbox = state == null ? null : state.queuedVote;
+ if (outbox == null) {
+ for (VoteTimeQueue candidate : getVoteCacheHandler().getTimeChangeQueue()) {
+ if (voteId.equals(candidate.getVoteId()) && candidate.isMultiProxyForwardingRequired()) {
+ outbox = candidate;
+ break;
+ }
+ }
+ }
+ if (outbox == null || !outbox.acknowledgeMultiProxyRetirement(recipient)) return;
+ outbox.setDeliveryStateDirty(true);
+ if (!persistTimeVoteDelivery(outbox)) {
+ scheduleTimeVoteRetry();
+ return;
+ }
+ scheduleTimeVoteRetry();
+ }
+
+ private boolean finishMultiProxyRetirement(LiveVoteRetryState retryState, VoteTimeQueue outbox) {
+ if (!outbox.hasCompletedMultiProxyRetirements()) {
+ if (multiProxyHandler != null) {
+ for (String recipient : outbox.getPendingMultiProxyRetirements()) {
+ multiProxyHandler.requestMultiProxyVoteRetirement(outbox.getVoteId(),
+ outbox.getMultiProxyOrigin(), recipient);
+ }
+ }
+ scheduleTimeVoteRetry();
+ return false;
+ }
+ return markMultiProxyForwardingHandled(retryState, outbox);
+ }
+
+ /** Persists a queued vote's multi-proxy side-effect fence before later retryable work. */
+ private boolean markMultiProxyForwardingHandled(LiveVoteRetryState retryState, VoteTimeQueue queuedVote) {
+ if (retryState != null) retryState.multiProxyForwardingHandled = true;
+ if (queuedVote == null) return true;
+ queuedVote.setMultiProxyForwardingHandled(true);
+ queuedVote.setMultiProxyCapabilityDiscoveryPending(false);
+ queuedVote.setDeliveryStateDirty(true);
+ return persistTimeVoteDelivery(queuedVote);
+ }
+
+ /**
+ * Creates the reward cache entry for a backend. Standalone proxy broadcast
+ * progress belongs to the single voter-keyed canonical state, not to every
+ * backend row. Marking the local row as already broadcast prevents it from
+ * emitting a second broadcast while its reward waits for the player.
+ */
+ protected OfflineBungeeVote createCachedRewardVote(UUID voteId, String player, String uuid, String service, long time,
+ boolean realVote, String text, boolean standaloneProxyBroadcast) {
+ return new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text,
+ standaloneProxyBroadcast, false, Collections.emptySet(), Collections.emptySet(), false,
+ Collections.emptyMap(), Collections.emptyMap());
+ }
+
+ private void markRewardJournalTargets(OfflineBungeeVote owner, Set targets) {
+ for (String server : targets) {
+ String key = owner.getVoteId() + ":reward:" + server;
+ owner.setHttpDeliveryId(REWARD_JOURNAL_TARGET_PREFIX + server,
+ UUID.nameUUIDFromBytes(key.getBytes(StandardCharsets.UTF_8)).toString());
+ }
+ }
+
+ private boolean isIncompleteRewardJournalOwner(OfflineBungeeVote vote) {
+ if (vote == null || vote.isRewardDelivered()) return false;
+ for (String key : vote.getHttpDeliveryIds().keySet()) {
+ if (key.startsWith(REWARD_JOURNAL_TARGET_PREFIX)) return true;
+ }
+ return false;
+ }
+
+ private String rewardJournalDeliveryId(OfflineBungeeVote owner, String server) {
+ String encoded = owner.getHttpDeliveryId(REWARD_JOURNAL_TARGET_PREFIX + server);
+ return encoded;
+ }
+
+ private boolean isRewardJournalOwner(OfflineBungeeVote vote) {
+ if (vote == null) return false;
+ for (String key : vote.getHttpDeliveryIds().keySet()) {
+ if (key.startsWith(REWARD_JOURNAL_TARGET_PREFIX)) return true;
+ }
+ return false;
+ }
+
+ /** Retries cleanup for an owner whose target rows were already materialized. */
+ private boolean retryCompletedRewardJournalOwner(String uuid, OfflineBungeeVote vote) {
+ if (uuid == null || vote == null || !vote.isRewardDelivered() || !isRewardJournalOwner(vote)) return false;
+ if (vote.isProxyBroadcastHandled() && !vote.isProxyBroadcastComplete()) return false;
+ if (!getVoteCacheHandler().tryRemoveOnlineVote(uuid, vote)) {
+ scheduleCachedVoteDeliveryRetry();
+ }
+ return true;
+ }
+
+ private boolean materializeRewardJournalOwner(String uuid, OfflineBungeeVote owner) {
+ for (Map.Entry entry : owner.getHttpDeliveryIds().entrySet()) {
+ if (!entry.getKey().startsWith(REWARD_JOURNAL_TARGET_PREFIX)) continue;
+ String normalizedServer = entry.getKey().substring(REWARD_JOURNAL_TARGET_PREFIX.length());
+ if (normalizedServer.isEmpty()) continue;
+ String server = normalizedServer;
+ for (String configured : getAllConfiguredServers()) {
+ if (configured.equalsIgnoreCase(normalizedServer)) {
+ server = configured;
+ break;
+ }
+ }
+ String deliveryId = entry.getValue();
+ OfflineBungeeVote reward = createCachedRewardVote(owner.getVoteId(), owner.getPlayerName(), owner.getUuid(),
+ owner.getService(), owner.getTime(), owner.isRealVote(), owner.getText(),
+ owner.isProxyBroadcastHandled());
+ reward.setHttpDeliveryId(server, deliveryId);
+ if (!getVoteCacheHandler().addServerVoteDurably(server, reward)) return false;
+ }
+ owner.setRewardDelivered(true);
+ owner.setDeliveryStateDirty(true);
+ if (!persistOnlineVoteDelivery(uuid, owner)) return false;
+ if (!owner.isProxyBroadcastHandled() || owner.isProxyBroadcastComplete()) {
+ if (!getVoteCacheHandler().tryRemoveOnlineVote(uuid, owner)) {
+ scheduleCachedVoteDeliveryRetry();
+ }
+ }
+ return true;
+ }
+
+ private OfflineBungeeVote createCachedRewardVote(OfflineBungeeVote deliveryState,
+ boolean standaloneProxyBroadcast) {
+ return new OfflineBungeeVote(deliveryState.getVoteId(), deliveryState.getPlayerName(), deliveryState.getUuid(),
+ deliveryState.getService(), deliveryState.getTime(), deliveryState.isRealVote(), deliveryState.getText(),
+ standaloneProxyBroadcast, false, Collections.emptySet(), Collections.emptySet(), false,
+ OfflineBungeeVote.decodeHttpDeliveryIds(deliveryState.encodeHttpDeliveryIds()), Collections.emptyMap());
+ }
+
+ protected boolean persistUncachedStandaloneBroadcast(String uuid, OfflineBungeeVote state,
+ boolean alreadyPersisted) {
+ if (alreadyPersisted || state == null || state.isProxyBroadcastComplete()) return true;
+ // This canonical row retries only the standalone broadcast. Reward delivery
+ // remains owned by the target-specific server/online cache entry.
+ state.setRewardDelivered(true);
+ state.setBroadcastForwarded(false);
+ if (getVoteCacheHandler().addOnlineVoteDurably(uuid, state)) return true;
+ boolean retained = getVoteCacheHandler().retainOnlineVoteForPersistenceRetry(uuid, state);
+ if (retained) scheduleCachedVoteDeliveryRetry();
+ return false;
+ }
+
+ protected boolean persistAndSendStandaloneBroadcast(String uuid, OfflineBungeeVote state,
+ Set remainingTargets, Set forwardedServers) {
+ if (!getVoteCacheHandler().addOnlineVoteDurably(uuid, state)) {
+ boolean retained = getVoteCacheHandler().retainOnlineVoteForPersistenceRetry(uuid, state);
+ if (retained) scheduleCachedVoteDeliveryRetry();
+ return false;
+ }
+ for (String target : remainingTargets) {
+ Set forwarded = sendProxyBroadcast(Collections.singleton(target), uuid, state.getPlayerName(),
+ state.getService(), state.getTime(), state.getText(), false, state);
+ forwardedServers.addAll(forwarded);
+ if (state.getBroadcastForwardedServers().addAll(forwarded) || state.isDeliveryStateDirty()) {
+ state.setBroadcastForwarded(state.isProxyBroadcastComplete());
+ if (!persistOnlineVoteDelivery(uuid, state)) return false;
+ }
+ }
+ return true;
+ }
private static final class PendingPresenceHandoff {
private UUID requestId;
private final UUID playerUuid;
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java
index 7420f0a49..adf9db1ad 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java
@@ -1,5 +1,7 @@
package com.bencodez.votingplugin.proxy;
+import java.util.UUID;
+
import com.bencodez.advancedcore.api.time.TimeType;
public class VotingPluginProxyCommand {
@@ -32,10 +34,22 @@ public String execute(String[] args) {
if (args.length >= 3) {
String user = args[1];
String site = args[2];
- plugin.vote(user, site, false, true, 0, null, null);
- return "&aVote sent for " + user + " on " + site;
+ UUID voteId;
+ try {
+ voteId = args.length >= 4 ? UUID.fromString(args[3]) : UUID.randomUUID();
+ } catch (IllegalArgumentException invalidId) {
+ return "&cUsage: vote [retry-id]";
+ }
+ try {
+ plugin.vote(user, site, false, true, 0, null, null, voteId);
+ return "&aVote sent for " + user + " on " + site;
+ } catch (IllegalArgumentException mismatchedRetry) {
+ return "&cRetry ID does not match that player and site.";
+ } catch (VotingPluginProxy.VoteRetryException retryable) {
+ return "&cVote could not be stored safely. Retry with: vote " + user + " " + site + " " + voteId;
+ }
}
- return "&cUsage: vote ";
+ return "&cUsage: vote [retry-id]";
case "forcetimechange":
if (args.length >= 2) {
@@ -47,6 +61,24 @@ public String execute(String[] args) {
case "status":
return handleStatusCommand();
+ case "httpcode":
+ if (args.length != 2) return "&cUsage: httpcode ";
+ try {
+ return "&aTemporary HTTP backend connection code (expires in 15 minutes and works once):\n&f"
+ + plugin.createHttpConnectionCode(args[1]);
+ } catch (IllegalArgumentException | IllegalStateException failure) {
+ return "&cThe secure HTTP transport is not running.";
+ }
+
+ case "httprevoke":
+ if (args.length != 2) return "&cUsage: httprevoke ";
+ try {
+ plugin.revokeHttpBackend(args[1]);
+ return "&aRevoked HTTP backend identity for " + args[1] + ". Generate a new connection code to re-enroll it.";
+ } catch (IllegalArgumentException | IllegalStateException failure) {
+ return "&cCould not revoke that HTTP backend identity.";
+ }
+
case "multiproxystatus":
plugin.getMultiProxyHandler().sendStatus();
return "&aSent status message across multi-proxy";
@@ -92,6 +124,8 @@ private String getHelpMessage() {
helpBuilder.append("/votingplugin vote - Send a vote\n");
helpBuilder.append("/votingplugin forcetimechange - Force a time change\n");
helpBuilder.append("/votingplugin status - Check connection status\n");
+ helpBuilder.append("/votingpluginproxy httpcode - Create a node-bound one-time HTTP connection code\n");
+ helpBuilder.append("/votingpluginproxy httprevoke - Revoke a backend identity before re-enrollment\n");
helpBuilder.append("/votingplugin multiproxystatus - Send status message across proxies\n");
helpBuilder.append("/votingplugin voteparty - Trigger or modify vote party\n");
return helpBuilder.toString();
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java
index 3c5eeff94..cd00c91e0 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java
@@ -241,6 +241,21 @@ default List getProxyBroadcastOfflineForwardServers() {
*/
public int getBungeePort();
+ /** Bind address for the single-port HTTP transport listener. */
+ default String getHttpHost() {
+ return "0.0.0.0";
+ }
+
+ /** Public HTTPS origin embedded in newly generated backend connection codes. */
+ default String getHttpPublicEndpoint() {
+ return "";
+ }
+
+ /** Listener port for the single-port HTTP transport. */
+ default int getHttpPort() {
+ return 1297;
+ }
+
/**
* Gets the plugin message channel.
*
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java
index 29becc646..598edac8a 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java
@@ -68,6 +68,12 @@ private VotingPluginWire() {
// =========================
public static final String SUB_CLEAR_VOTE = "ClearVote";
public static final String SUB_CLEAR_VOTE_PRIMARY = "ClearVotePrimary";
+ /** Additive acknowledgement for the reliable multi-proxy vote envelope. */
+ public static final String SUB_MULTI_PROXY_VOTE_ACK = "MultiProxyVoteAck";
+ public static final String SUB_MULTI_PROXY_VOTE_RETIRE = "MultiProxyVoteRetire";
+ public static final String SUB_MULTI_PROXY_VOTE_RETIRE_ACK = "MultiProxyVoteRetireAck";
+ /** Additive capability advertisement for durable multi-proxy acknowledgements. */
+ public static final String SUB_MULTI_PROXY_CAPABILITIES = "MultiProxyCapabilities";
// =========================
// Field keys (public where referenced externally)
@@ -102,6 +108,11 @@ private VotingPluginWire() {
public static final String K_BUNGEE_BROADCAST = "bungeeBroadcast";
public static final String K_NUM = "num";
public static final String K_NUMBER_OF_VOTES = "numberOfVotes";
+ /** Origin and receiving proxy names for reliable multi-proxy delivery. */
+ public static final String K_MULTI_PROXY_ORIGIN = "multiProxyOrigin";
+ public static final String K_MULTI_PROXY_RECIPIENT = "multiProxyRecipient";
+ public static final String K_MULTI_PROXY_ACK_VERSION = "multiProxyAckVersion";
+ public static final String K_MULTI_PROXY_CAPABILITY_REPLY = "multiProxyCapabilityReply";
// VoteUpdate extras
public static final String K_PLAYER_UUID = "playerUuid";
@@ -140,6 +151,31 @@ public static JsonEnvelope voteOnline(String player, String uuid, String service
.put(K_NUM, num).put(K_NUMBER_OF_VOTES, numberOfVotes).build();
}
+ /**
+ * Builds a multi-proxy vote with an additive sender identity for its durable
+ * acknowledgement. Older receivers safely ignore the extra field.
+ */
+ public static JsonEnvelope multiProxyVote(String player, String uuid, String service, long time, boolean wasOnline,
+ boolean realVote, String totals, UUID voteId, boolean manageTotals, boolean bungeeBroadcast, int num,
+ int numberOfVotes, String origin) {
+ return base(SUB_VOTE).put(K_PLAYER, safe(player)).put(K_UUID, safe(uuid)).put(K_SERVICE, safe(service))
+ .put(K_TIME, time).put(K_WAS_ONLINE, wasOnline).put(K_REAL_VOTE, realVote).put(K_TOTALS, safe(totals))
+ .put(K_VOTE_ID, voteId == null ? "" : voteId.toString()).put(K_SET_TOTALS, true)
+ .put(K_MANAGE_TOTALS, manageTotals).put(K_BUNGEE_BROADCAST, bungeeBroadcast).put(K_NUM, num)
+ .put(K_NUMBER_OF_VOTES, numberOfVotes).put(K_MULTI_PROXY_ORIGIN, safe(origin)).build();
+ }
+
+ /** Reliable multi-proxy variant of {@link #voteOnline}. */
+ public static JsonEnvelope multiProxyVoteOnline(String player, String uuid, String service, long time,
+ boolean wasOnline, boolean realVote, String totals, UUID voteId, boolean manageTotals,
+ boolean bungeeBroadcast, int num, int numberOfVotes, String origin) {
+ return base(SUB_VOTE_ONLINE).put(K_PLAYER, safe(player)).put(K_UUID, safe(uuid)).put(K_SERVICE, safe(service))
+ .put(K_TIME, time).put(K_WAS_ONLINE, wasOnline).put(K_REAL_VOTE, realVote).put(K_TOTALS, safe(totals))
+ .put(K_VOTE_ID, voteId == null ? "" : voteId.toString()).put(K_SET_TOTALS, true)
+ .put(K_MANAGE_TOTALS, manageTotals).put(K_BUNGEE_BROADCAST, bungeeBroadcast).put(K_NUM, num)
+ .put(K_NUMBER_OF_VOTES, numberOfVotes).put(K_MULTI_PROXY_ORIGIN, safe(origin)).build();
+ }
+
public static JsonEnvelope voteDelayRejected(String player, String uuid, String service, boolean wasOnline) {
return base(SUB_VOTE_DELAY_REJECTED).put(K_PLAYER, safe(player)).put(K_UUID, safe(uuid))
.put(K_SERVICE, safe(service)).put(K_WAS_ONLINE, wasOnline).build();
@@ -366,6 +402,36 @@ public static JsonEnvelope clearVotePrimary(String uuid, String player, String s
.put(K_SERVER, safe(server)).build();
}
+ /** Acknowledges durable completion to the originating proxy. */
+ public static JsonEnvelope multiProxyVoteAck(UUID voteId, String origin, String recipient) {
+ return base(SUB_MULTI_PROXY_VOTE_ACK).put(K_VOTE_ID, voteId == null ? "" : voteId.toString())
+ .put(K_MULTI_PROXY_ORIGIN, safe(origin)).put(K_MULTI_PROXY_RECIPIENT, safe(recipient)).build();
+ }
+
+ /** Requests deletion of a receiver fence after every vote ACK is durable. */
+ public static JsonEnvelope multiProxyVoteRetire(UUID voteId, String origin, String recipient) {
+ return base(SUB_MULTI_PROXY_VOTE_RETIRE).put(K_VOTE_ID, voteId == null ? "" : voteId.toString())
+ .put(K_MULTI_PROXY_ORIGIN, safe(origin)).put(K_MULTI_PROXY_RECIPIENT, safe(recipient)).build();
+ }
+
+ /** Confirms idempotent receiver-fence retirement to the originating proxy. */
+ public static JsonEnvelope multiProxyVoteRetireAck(UUID voteId, String origin, String recipient) {
+ return base(SUB_MULTI_PROXY_VOTE_RETIRE_ACK).put(K_VOTE_ID, voteId == null ? "" : voteId.toString())
+ .put(K_MULTI_PROXY_ORIGIN, safe(origin)).put(K_MULTI_PROXY_RECIPIENT, safe(recipient)).build();
+ }
+
+ /** Advertises support for the additive durable multi-proxy acknowledgement protocol. */
+ public static JsonEnvelope multiProxyCapabilities(String recipient, int acknowledgementVersion) {
+ return multiProxyCapabilities(recipient, acknowledgementVersion, false);
+ }
+
+ /** Capability response used to complete a bounded bidirectional handshake. */
+ public static JsonEnvelope multiProxyCapabilities(String recipient, int acknowledgementVersion, boolean reply) {
+ return base(SUB_MULTI_PROXY_CAPABILITIES).put(K_MULTI_PROXY_RECIPIENT, safe(recipient))
+ .put(K_MULTI_PROXY_ACK_VERSION, acknowledgementVersion)
+ .put(K_MULTI_PROXY_CAPABILITY_REPLY, reply).build();
+ }
+
// =========================
// Readers (decode)
// =========================
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java
index ee1ff540a..4aa394cca 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java
@@ -209,9 +209,24 @@ public String getBungeeMethod() {
}
@Override
- public int getBungeePort() {
- return getData().getInt("BungeeServer.Port", 1297);
- }
+ public int getBungeePort() {
+ return getData().getInt("BungeeServer.Port", 1297);
+ }
+
+ @Override
+ public String getHttpHost() {
+ return getData().getString("HTTP.Host", "0.0.0.0");
+ }
+
+ @Override
+ public String getHttpPublicEndpoint() {
+ return getData().getString("HTTP.PublicEndpoint", "");
+ }
+
+ @Override
+ public int getHttpPort() {
+ return getData().getInt("HTTP.Port", 1297);
+ }
@Override
public boolean getDebug() {
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java
index 0291a0a48..2d369e1d4 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java
@@ -1,19 +1,29 @@
package com.bencodez.votingplugin.proxy.bungee;
import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Base64;
import java.util.Collection;
+import java.util.Locale;
import com.bencodez.simpleapi.file.BungeeJsonFile;
import com.bencodez.votingplugin.proxy.OfflineBungeeVote;
import com.bencodez.votingplugin.proxy.cache.DataNode;
import com.bencodez.votingplugin.proxy.cache.GsonDataNode;
-import com.bencodez.votingplugin.proxy.cache.IVoteCache;
-import com.bencodez.votingplugin.timequeue.VoteTimeQueue;
+import com.bencodez.votingplugin.proxy.cache.IVoteCache;
+import com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects;
+import com.bencodez.votingplugin.timequeue.VoteTimeQueue;
+import com.bencodez.votingplugin.util.DurableFiles;
+import com.google.gson.GsonBuilder;
/**
* JSON file-based vote cache for Bungee.
*/
-public class BungeeJsonVoteCache extends BungeeJsonFile implements IVoteCache {
+public class BungeeJsonVoteCache extends BungeeJsonFile implements IVoteCache {
private VotingPluginBungee bungee;
/**
@@ -27,7 +37,7 @@ public BungeeJsonVoteCache(VotingPluginBungee bungee) {
initialize();
}
- private void initialize() {
+ private void initialize() {
if (!bungee.getDataFolder().exists()) {
bungee.getDataFolder().mkdir();
}
@@ -43,11 +53,44 @@ public void addTimedVote(int num, VoteTimeQueue voteTimedQueue) {
setString(path + ".VoteId", voteTimedQueue.getVoteId() == null ? null : voteTimedQueue.getVoteId().toString());
setString(path + ".UUID", voteTimedQueue.getUuid());
setBoolean(path + ".ProxyBroadcastHandled", voteTimedQueue.isProxyBroadcastHandled());
- setString(path + ".Totals", voteTimedQueue.getTotals());
- setBoolean(path + ".Processed", voteTimedQueue.isProcessed());
- setString(path + ".BroadcastTargets", voteTimedQueue.encodeBroadcastTargets());
- setString(path + ".BroadcastForwardedServers", voteTimedQueue.encodeBroadcastForwardedServers());
- }
+ setString(path + ".Totals", voteTimedQueue.getTotals());
+ setBoolean(path + ".Processed", voteTimedQueue.isProcessed());
+ setBoolean(path + ".MultiProxyForwardingHandled", voteTimedQueue.isMultiProxyForwardingHandled());
+ setBoolean(path + ".MultiProxyForwardingRequired", voteTimedQueue.isMultiProxyForwardingRequired());
+ setBoolean(path + ".MultiProxyCapabilityDiscoveryPending",
+ voteTimedQueue.isMultiProxyCapabilityDiscoveryPending());
+ setBoolean(path + ".RealVote", voteTimedQueue.isRealVote());
+ setString(path + ".MultiProxyOrigin", voteTimedQueue.getMultiProxyOrigin());
+ setBoolean(path + ".MultiProxyCompletionPending", voteTimedQueue.isMultiProxyCompletionPending());
+ setString(path + ".MultiProxyRecipients", voteTimedQueue.encodeMultiProxyRecipients());
+ setString(path + ".MultiProxyAcknowledgedServers", voteTimedQueue.encodeMultiProxyAcknowledgedServers());
+ setString(path + ".MultiProxyLegacyPendingRecipients",
+ voteTimedQueue.encodeMultiProxyLegacyPendingRecipients());
+ setString(path + ".BroadcastTargets", voteTimedQueue.encodeBroadcastTargets());
+ setString(path + ".BroadcastForwardedServers", voteTimedQueue.encodeBroadcastForwardedServers());
+ setString(path + ".HttpBroadcastDeliveryIds", voteTimedQueue.encodeHttpBroadcastDeliveryIds());
+ }
+
+ @Override
+ public java.nio.file.Path getStoragePath() {
+ return getFile().toPath();
+ }
+
+ @Override
+ public synchronized void saveDurably() throws IOException {
+ Path target = getStoragePath().toAbsolutePath().normalize();
+ Path parent = target.getParent();
+ if (parent == null) throw new IOException("Vote cache has no parent directory");
+ Files.createDirectories(parent);
+ Path staged = Files.createTempFile(parent, target.getFileName().toString(), ".tmp");
+ try {
+ Files.writeString(staged, new GsonBuilder().setPrettyPrinting().create().toJson(getConf()),
+ StandardCharsets.UTF_8);
+ DurableFiles.publishStagedFile(staged, target);
+ } finally {
+ Files.deleteIfExists(staged);
+ }
+ }
public void addVote(String server, int num, OfflineBungeeVote voteData) {
String path = "VoteCache." + server + "." + num;
@@ -61,8 +104,10 @@ public void addVote(String server, int num, OfflineBungeeVote voteData) {
setBoolean(path + ".BroadcastForwarded", voteData.isBroadcastForwarded());
setBoolean(path + ".ProxyBroadcastHandled", voteData.isProxyBroadcastHandled());
setString(path + ".BroadcastTargets", voteData.encodeBroadcastTargets());
- setString(path + ".BroadcastForwardedServers", voteData.encodeBroadcastForwardedServers());
- setBoolean(path + ".RewardDelivered", voteData.isRewardDelivered());
+ setString(path + ".BroadcastForwardedServers", voteData.encodeBroadcastForwardedServers());
+ setBoolean(path + ".RewardDelivered", voteData.isRewardDelivered());
+ setString(path + ".HttpDeliveryIds", voteData.encodeHttpDeliveryIds());
+ setString(path + ".HttpBroadcastDeliveryIds", voteData.encodeHttpBroadcastDeliveryIds());
}
public void addVoteOnline(String player, int num, OfflineBungeeVote voteData) {
@@ -77,8 +122,10 @@ public void addVoteOnline(String player, int num, OfflineBungeeVote voteData) {
setBoolean(path + ".BroadcastForwarded", voteData.isBroadcastForwarded());
setBoolean(path + ".ProxyBroadcastHandled", voteData.isProxyBroadcastHandled());
setString(path + ".BroadcastTargets", voteData.encodeBroadcastTargets());
- setString(path + ".BroadcastForwardedServers", voteData.encodeBroadcastForwardedServers());
- setBoolean(path + ".RewardDelivered", voteData.isRewardDelivered());
+ setString(path + ".BroadcastForwardedServers", voteData.encodeBroadcastForwardedServers());
+ setBoolean(path + ".RewardDelivered", voteData.isRewardDelivered());
+ setString(path + ".HttpDeliveryIds", voteData.encodeHttpDeliveryIds());
+ setString(path + ".HttpBroadcastDeliveryIds", voteData.encodeHttpBroadcastDeliveryIds());
}
public void clearData() {
@@ -128,6 +175,35 @@ public void removeTimedVotes() {
public int getVotePartyCache(String server) {
return getInt("VoteParty.Cache." + server, 0);
}
+
+ @Override
+ public Collection getPendingVotePartyRewardServers() {
+ Collection encoded = getKeys("VoteParty.PendingRewards");
+ Collection servers = new ArrayList<>();
+ if (encoded != null) for (String key : encoded) try {
+ servers.add(decodeServerKey(key));
+ } catch (IllegalArgumentException malformedKey) {
+ // Preserve later valid deliveries when one persisted key is malformed.
+ }
+ return servers;
+ }
+
+ @Override
+ public Collection getPendingVotePartyRewardIds(String server) {
+ return getKeys("VoteParty.PendingRewards." + encodeServerKey(server));
+ }
+
+ @Override
+ public PendingVotePartyProxyEffects getPendingVotePartyProxyEffects() {
+ return new PendingVotePartyProxyEffects(getString("VoteParty.PendingProxyEffects.Broadcast", ""),
+ getStringList("VoteParty.PendingProxyEffects.Commands", java.util.List.of()));
+ }
+
+ @Override
+ public PendingVotePartyProxyEffects getQuarantinedVotePartyProxyEffects() {
+ return new PendingVotePartyProxyEffects(getString("VoteParty.QuarantinedProxyEffects.Broadcast", ""),
+ getStringList("VoteParty.QuarantinedProxyEffects.Commands", java.util.List.of()));
+ }
public int getVotePartyCurrentVotes() {
return getInt("VoteParty.CurrentVotes", 0);
@@ -140,6 +216,47 @@ public int getVotePartyInreaseVotesRequired() {
public void setVotePartyCache(String server, int amount) {
setInt("VoteParty.Cache." + server, amount);
}
+
+ @Override
+ public void setPendingVotePartyReward(String server, String deliveryId, boolean pending) {
+ String serverPath = "VoteParty.PendingRewards." + encodeServerKey(server);
+ String path = serverPath + "." + deliveryId;
+ if (pending) setBoolean(path, true);
+ else {
+ setString(path, null);
+ Collection remaining = getKeys(serverPath);
+ if (remaining == null || remaining.isEmpty()) setString(serverPath, null);
+ }
+ }
+
+ @Override
+ public void setPendingVotePartyProxyEffects(PendingVotePartyProxyEffects effects) {
+ if (effects.isEmpty()) {
+ remove("VoteParty.PendingProxyEffects");
+ return;
+ }
+ setString("VoteParty.PendingProxyEffects.Broadcast", effects.broadcast());
+ setStringList("VoteParty.PendingProxyEffects.Commands", effects.commands());
+ }
+
+ @Override
+ public void setQuarantinedVotePartyProxyEffects(PendingVotePartyProxyEffects effects) {
+ if (effects.isEmpty()) {
+ remove("VoteParty.QuarantinedProxyEffects");
+ return;
+ }
+ setString("VoteParty.QuarantinedProxyEffects.Broadcast", effects.broadcast());
+ setStringList("VoteParty.QuarantinedProxyEffects.Commands", effects.commands());
+ }
+
+ private static String encodeServerKey(String server) {
+ return Base64.getUrlEncoder().withoutPadding()
+ .encodeToString(server.toLowerCase(Locale.ROOT).getBytes(StandardCharsets.UTF_8));
+ }
+
+ private static String decodeServerKey(String server) {
+ return new String(Base64.getUrlDecoder().decode(server), StandardCharsets.UTF_8);
+ }
public void setVotePartyCurrentVotes(int amount) {
setInt("VoteParty.CurrentVotes", amount);
@@ -176,12 +293,12 @@ public void removeServerVote(String server, String uuid) {
return;
}
// search for vote with uuid and remove it
- for (String num : votes) {
- GsonDataNode node = getServerVotes(server, num);
+ for (String num : votes) {
+ GsonDataNode node = getServerVotes(server, num);
if (node == null) {
continue;
}
- DataNode uuidNode = node.get("UUID");
+ DataNode uuidNode = node.get("UUID");
if (uuidNode == null) {
continue;
}
@@ -192,18 +309,30 @@ public void removeServerVote(String server, String uuid) {
}
}
- @Override
- public void removeVote(String server, OfflineBungeeVote vote) {
- Collection votes = getServerVotes(server);
+ @Override
+ public void removeVote(String server, OfflineBungeeVote vote) {
+ if (vote.getServerVoteCacheJsonKey() != null) {
+ setString("VoteCache." + server + "." + vote.getServerVoteCacheJsonKey(), null);
+ return;
+ }
+ Collection votes = getServerVotes(server);
if (votes == null) {
return;
}
for (String num : votes) {
GsonDataNode node = getServerVotes(server, num);
- if (node == null) {
- continue;
- }
- DataNode uuidNode = node.get("UUID");
+ if (node == null) {
+ continue;
+ }
+ DataNode voteIdNode = node.has("VoteId") ? node.get("VoteId")
+ : node.has("VoteID") ? node.get("VoteID") : null;
+ if (vote.getVoteId() != null && voteIdNode != null) {
+ if (vote.getVoteId().toString().equals(voteIdNode.asString())) {
+ setString("VoteCache." + server + "." + num, null);
+ }
+ continue;
+ }
+ DataNode uuidNode = node.get("UUID");
DataNode serviceNode = node.get("Service");
DataNode timeNode = node.get("Time");
if (uuidNode == null || serviceNode == null || timeNode == null) {
@@ -219,9 +348,13 @@ public void removeVote(String server, OfflineBungeeVote vote) {
}
}
- @Override
- public void removeOnlineVote(OfflineBungeeVote vote) {
- Collection players = getPlayers();
+ @Override
+ public void removeOnlineVote(OfflineBungeeVote vote) {
+ if (vote.getOnlineVoteCacheJsonKey() != null && vote.getUuid() != null) {
+ setString("OnlineCache." + vote.getUuid() + "." + vote.getOnlineVoteCacheJsonKey(), null);
+ return;
+ }
+ Collection players = getPlayers();
if (players == null) {
return;
}
@@ -230,12 +363,20 @@ public void removeOnlineVote(OfflineBungeeVote vote) {
if (onlineVotes == null) {
continue;
}
- for (String num : onlineVotes) {
- GsonDataNode node = getOnlineVotes(player, num);
+ for (String num : onlineVotes) {
+ GsonDataNode node = getOnlineVotes(player, num);
if (node == null) {
continue;
}
- DataNode uuidNode = node.get("UUID");
+ DataNode voteIdNode = node.has("VoteId") ? node.get("VoteId")
+ : node.has("VoteID") ? node.get("VoteID") : null;
+ if (vote.getVoteId() != null && voteIdNode != null) {
+ if (vote.getVoteId().toString().equals(voteIdNode.asString())) {
+ setString("OnlineCache." + player + "." + num, null);
+ }
+ continue;
+ }
+ DataNode uuidNode = node.get("UUID");
DataNode serviceNode = node.get("Service");
DataNode timeNode = node.get("Time");
if (uuidNode == null || serviceNode == null || timeNode == null) {
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VoteEventBungee.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VoteEventBungee.java
index 79c41a464..f01f01cb1 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VoteEventBungee.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VoteEventBungee.java
@@ -1,6 +1,10 @@
package com.bencodez.votingplugin.proxy.bungee;
+import java.util.concurrent.TimeUnit;
+import java.util.UUID;
+
import com.bencodez.votingplugin.util.MinecraftUsernameValidator;
+import com.bencodez.votingplugin.proxy.VotingPluginProxy.VoteRetryException;
import com.vexsoftware.votifier.bungee.events.VotifierEvent;
import com.vexsoftware.votifier.model.Vote;
@@ -26,25 +30,44 @@ public VoteEventBungee(VotingPluginBungee plugin) {
*/
@EventHandler
public void onVote(VotifierEvent event) {
- plugin.getProxy().getScheduler().runAsync(plugin, new Runnable() {
-
- @SuppressWarnings("deprecation")
- @Override
- public void run() {
- Vote vote = event.getVote();
- String serviceSite = vote.getServiceName();
- plugin.getLogger().info("Vote received " + MinecraftUsernameValidator.sanitizeForLog(vote.getUsername())
- + " from service site " + MinecraftUsernameValidator.sanitizeForLog(serviceSite));
-
- if (serviceSite.isEmpty()) {
- serviceSite = "Empty";
- vote.setServiceName(serviceSite);
- }
+ Vote vote = event.getVote();
+ String serviceName = vote.getServiceName();
+ String serviceSite = serviceName == null || serviceName.isEmpty() ? "Empty" : serviceName;
+ plugin.getProxy().getScheduler().runAsync(plugin,
+ new RetryingVote(vote.getUsername(), serviceSite));
- plugin.getVotingPluginProxy().vote(vote.getUsername(), serviceSite, true, true, 0, null, null);
- }
- });
+ }
+ private final class RetryingVote implements Runnable {
+ private static final int MAX_ATTEMPTS = 12;
+ private final String player;
+ private final String service;
+ private final UUID voteId = UUID.randomUUID();
+ private int attempts;
+
+ private RetryingVote(String player, String service) {
+ this.player = player;
+ this.service = service;
+ }
+
+ @Override
+ public void run() {
+ plugin.getLogger().info("Vote received " + MinecraftUsernameValidator.sanitizeForLog(player)
+ + " from service site " + MinecraftUsernameValidator.sanitizeForLog(service));
+ try {
+ plugin.getVotingPluginProxy().vote(player, service, true, true, 0, null, null, voteId);
+ } catch (VoteRetryException retryable) {
+ attempts++;
+ if (attempts < MAX_ATTEMPTS) {
+ plugin.getLogger().warning("Vote processing is waiting for durable storage; retrying shortly");
+ plugin.getProxy().getScheduler().schedule(plugin, this, 5, TimeUnit.SECONDS);
+ } else {
+ plugin.getVotingPluginProxy().abandonLiveVoteRetry(voteId);
+ plugin.getLogger().severe("Vote processing exhausted bounded durable-storage retries for "
+ + MinecraftUsernameValidator.sanitizeForLog(player));
+ }
+ }
+ }
}
}
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java
index bd644c474..111aaf689 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java
@@ -7,16 +7,18 @@
import java.io.Reader;
import java.net.URL;
import java.security.CodeSource;
+import java.util.Collection;
import java.util.HashSet;
import java.util.Locale;
import java.util.Map.Entry;
import java.util.Queue;
import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BooleanSupplier;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
@@ -312,8 +314,8 @@ public void onPluginMessage(PluginMessageEvent ev) {
* @param loadMysql true to fully reinitialize MySQL and rebuild the proxy
* runtime, false for a config/runtime-only reload
*/
- public void reloadPlugin(boolean loadMysql) {
- synchronized (reloadLock) {
+ public void reloadPlugin(boolean loadMysql) {
+ synchronized (reloadLock) {
reloading = true;
final String oldChannel = (config != null) ? config.getPluginMessageChannel() : null;
@@ -376,11 +378,25 @@ public void reloadPlugin(boolean loadMysql) {
return;
}
- // =========================
- // FULL RELOAD (WITH MYSQL)
- // =========================
-
- // Save caches best-effort before teardown
+ // =========================
+ // FULL RELOAD (WITH MYSQL)
+ // =========================
+ // Keep the old listener and its original runtime alive when a changed
+ // config selects a non-HTTP transport while its durable HTTP queue is not
+ // empty. Recreating first would make the new (possibly blank/changed)
+ // HTTP endpoint own that queue and can prevent it from draining.
+ if (votingPluginProxy != null
+ && votingPluginProxy.requiresHttpRetentionCheckBeforeRuntimeReplacement()) {
+ votingPluginProxy.reload();
+ if (votingPluginProxy.isRetainingHttpTransportForDeferredReconciliation()) {
+ schedulePlatformTasks();
+ reloading = false;
+ drainQueuedPluginMessagesAfterReloadLock();
+ return;
+ }
+ }
+
+ // Save caches best-effort before teardown
try {
if (voteCacheFile != null) {
voteCacheFile.save();
@@ -485,8 +501,28 @@ public void reloadPlugin(boolean loadMysql) {
try {
getVotingPluginProxy().sendServerNameMessage();
} catch (Exception ignored) {
- }
- }
+ }
+ }
+
+ /** The retention branch returns from inside reloadLock; drain only after that lock is released. */
+ private void drainQueuedPluginMessagesAfterReloadLock() {
+ Thread drain = new Thread(() -> {
+ synchronized (reloadLock) {
+ // Acquire/release establishes that the returning reload has left its lock.
+ }
+ drainQueuedPluginMessages();
+ }, "VotingPlugin-Bungee-Reload-Queue-Drain");
+ drain.setDaemon(true);
+ drain.start();
+ }
+
+ /** Runs a scheduled deferred replacement only if it is still current while holding reloadLock. */
+ private void reloadPluginIfCurrent(boolean loadMysql, BooleanSupplier stillCurrent) {
+ synchronized (reloadLock) {
+ if (!stillCurrent.getAsBoolean()) return;
+ reloadPlugin(loadMysql);
+ }
+ }
/**
* Creates a new VotingPluginProxy instance wired to this platform.
@@ -623,6 +659,26 @@ public int getVoteCachePrevWeek() {
public int getVoteCacheVotePartyIncreaseVotesRequired() {
return voteCacheFile.getVotePartyInreaseVotesRequired();
}
+
+ @Override
+ public Collection getVoteCachePendingVotePartyServers() {
+ return voteCacheFile.getPendingVotePartyRewardServers();
+ }
+
+ @Override
+ public Collection getVoteCachePendingVotePartyRewardIds(String server) {
+ return voteCacheFile.getPendingVotePartyRewardIds(server);
+ }
+
+ @Override
+ public com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects getVoteCachePendingVotePartyProxyEffects() {
+ return voteCacheFile.getPendingVotePartyProxyEffects();
+ }
+
+ @Override
+ public com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects getVoteCacheQuarantinedVotePartyProxyEffects() {
+ return voteCacheFile.getQuarantinedVotePartyProxyEffects();
+ }
@Override
public boolean isPlayerOnline(String playerName) {
@@ -661,15 +717,30 @@ public void runAsync(Runnable run) {
runAsyncNow(run);
}
- @Override
- public void runConsoleCommand(String command) {
- getProxy().getPluginManager().dispatchCommand(getProxy().getConsole(), command);
- }
+ @Override
+ public void runConsoleCommand(String command) {
+ getProxy().getPluginManager().dispatchCommand(getProxy().getConsole(), command);
+ }
+
+ @Override
+ protected java.util.concurrent.CompletableFuture runVotePartyConsoleCommand(String command) {
+ if (!getProxy().getPluginManager().dispatchCommand(getProxy().getConsole(), command)) {
+ return java.util.concurrent.CompletableFuture.failedFuture(
+ new IllegalStateException("Bungee declined the vote-party proxy command"));
+ }
+ return java.util.concurrent.CompletableFuture.completedFuture(null);
+ }
@Override
public void saveVoteCacheFile() {
voteCacheFile.save();
}
+
+ @Override
+ public void saveVotePartyStateDurably() throws java.io.IOException {
+ com.bencodez.votingplugin.proxy.cache.VotePartyCacheDurability.saveAndVerify(
+ new File(getDataFolder(), "votecache.json").toPath(), voteCacheFile);
+ }
@Override
public boolean sendPluginMessageData(String server, String channel, byte[] data, boolean queue) {
@@ -718,18 +789,40 @@ public void setVoteCacheVotePartyCurrentVotes(int votes) {
public void setVoteCacheVotePartyIncreaseVotesRequired(int votes) {
voteCacheFile.setVotePartyInreaseVotesRequired(votes);
}
+
+ @Override
+ public void setVoteCachePendingVotePartyReward(String server, String deliveryId, boolean pending) {
+ voteCacheFile.setPendingVotePartyReward(server, deliveryId, pending);
+ }
+
+ @Override
+ public void setVoteCachePendingVotePartyProxyEffects(
+ com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects effects) {
+ voteCacheFile.setPendingVotePartyProxyEffects(effects);
+ }
+
+ @Override
+ public void setVoteCacheQuarantinedVotePartyProxyEffects(
+ com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects effects) {
+ voteCacheFile.setQuarantinedVotePartyProxyEffects(effects);
+ }
@Override
public void warn(String message) {
getLogger().warning(message);
}
- @Override
+ @Override
public void reloadCore(boolean mysql) {
- // mysql==true should do full reloadall behavior on the platform
+ // mysql==true should do full reloadall behavior on the platform
reloadPlugin(mysql);
}
+ @Override
+ protected void reloadDeferredHttpTransportCore(long generation) {
+ reloadPluginIfCurrent(true, () -> isDeferredHttpTransportGenerationCurrent(generation));
+ }
+
@Override
public void reloadControlConfiguration() throws Exception {
config.loadControlConfiguration();
diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java
index 03b6e4977..eeb6ec592 100644
--- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java
+++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java
@@ -1,6 +1,14 @@
package com.bencodez.votingplugin.proxy.cache;
import java.util.Collection;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+
+import com.bencodez.votingplugin.util.DurableFiles;
import com.bencodez.votingplugin.proxy.OfflineBungeeVote;
import com.bencodez.votingplugin.timequeue.VoteTimeQueue;
@@ -10,6 +18,97 @@
*/
public interface IVoteCache {
+ static final String EMERGENCY_JOURNAL_MARKER_FILE_SUFFIX = ".vote-emergency-journal";
+ static final String EMERGENCY_JOURNAL_MARKER_VERSION = "1";
+
+ /**
+ * Returns the marker file used for emergency-journal recovery coordination.
+ *
+ * @return marker file path, or {@code null} when no file-backed cache exists
+ */
+ default Path getEmergencyJournalMarkerPath() {
+ Path storagePath = getStoragePath();
+ if (storagePath == null) {
+ return null;
+ }
+ return storagePath.resolveSibling(storagePath.getFileName() + EMERGENCY_JOURNAL_MARKER_FILE_SUFFIX);
+ }
+
+ /**
+ * Returns whether this JSON cache has a marker that indicates emergency journal
+ * compatibility for MySQL recovery. Legacy files without this marker are not
+ * treated as emergency recoverables.
+ *
+ * @return true when the marker file exists
+ */
+ default boolean hasEmergencyJournalMarker() {
+ Path markerPath = getEmergencyJournalMarkerPath();
+ if (markerPath == null) {
+ return false;
+ }
+ try {
+ return Files.isRegularFile(markerPath, LinkOption.NOFOLLOW_LINKS) && !Files.isSymbolicLink(markerPath)
+ && Files.size(markerPath) <= 8
+ && EMERGENCY_JOURNAL_MARKER_VERSION.equals(Files.readString(markerPath, StandardCharsets.UTF_8).trim());
+ } catch (IOException ignored) {
+ return false;
+ }
+ }
+
+ /**
+ * Writes the emergency-journal marker file used to indicate that this cache has
+ * participated in emergency journal recovery flow.
+ *
+ * @throws IOException when marker publication fails
+ */
+ default void markEmergencyJournalUsed() throws IOException {
+ Path markerPath = getEmergencyJournalMarkerPath();
+ if (markerPath == null) {
+ throw new IOException("Vote cache has no emergency journal marker path");
+ }
+ Path normalizedMarker = markerPath.toAbsolutePath().normalize();
+ Path parent = normalizedMarker.getParent();
+ if (parent != null && Files.isSymbolicLink(parent)) {
+ throw new IOException("Emergency journal marker parent is a symbolic link");
+ }
+ if (Files.isSymbolicLink(normalizedMarker)) {
+ throw new IOException("Emergency journal marker is a symbolic link");
+ }
+ if (parent != null) {
+ Files.createDirectories(parent);
+ }
+ Path staged = Files.createTempFile(parent, normalizedMarker.getFileName().toString(), ".journal");
+ try {
+ Files.writeString(staged, EMERGENCY_JOURNAL_MARKER_VERSION, StandardCharsets.UTF_8, StandardOpenOption.CREATE,
+ StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
+ DurableFiles.publishStagedFile(staged, normalizedMarker);
+ } finally {
+ Files.deleteIfExists(staged);
+ }
+ }
+
+ /**
+ * Returns the file backing this JSON cache, when one exists.
+ *
+ * @return backing file path, or {@code null} for non-file implementations
+ */
+ default Path getStoragePath() {
+ return null;
+ }
+
+ /** Persists the complete cache and forces it to stable storage. */
+ default void saveDurably() throws IOException {
+ Path path = getStoragePath();
+ if (path == null) throw new IOException("Vote cache has no durable storage path");
+ save();
+ DurableFiles.forceFile(path);
+ try {
+ DurableFiles.forceDirectory(path.toAbsolutePath().normalize().getParent());
+ } catch (IOException failure) {
+ throw new DurableFiles.PublishedException(failure);
+ }
+ }
+
/**
* Adds a timed vote to the cache.
*
@@ -117,6 +216,15 @@ public interface IVoteCache {
*/
int getVotePartyCache(String server);
+ /** Returns backend IDs with persisted, undelivered vote-party rewards. */
+ Collection